From 7fa3da5ae827944d18d64a3fbe96bce4638db301 Mon Sep 17 00:00:00 2001 From: greyllmmoder Date: Sun, 5 Apr 2026 13:46:27 +0530 Subject: [PATCH] chore: add conformance evidence tooling and release gates --- .github/workflows/ci.yml | 37 +++- .github/workflows/release-gate.yml | 55 +++++ .gitignore | 3 + MANIFEST.in | 1 + PRODUCTION_CHECKLIST.md | 3 + README.md | 10 + docs/ISO_IEC_24778_TRACEABILITY.md | 41 ++++ scripts/conformance_report.py | 336 +++++++++++++++++++++++++++++ tests/test_conformance_report.py | 41 ++++ 9 files changed, 526 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release-gate.yml create mode 100644 docs/ISO_IEC_24778_TRACEABILITY.md create mode 100644 scripts/conformance_report.py create mode 100644 tests/test_conformance_report.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f789203..f32699f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,9 @@ on: push: pull_request: +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -29,6 +32,38 @@ jobs: run: | pytest -q + conformance: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev,image,pdf]" + + - name: Generate conformance evidence + run: | + python scripts/conformance_report.py \ + --report conformance_report.md \ + --json conformance_report.json \ + --matrix-report compat_matrix_report.md + + - name: Upload conformance artifacts + uses: actions/upload-artifact@v4 + with: + name: conformance-evidence + path: | + conformance_report.md + conformance_report.json + compat_matrix_report.md + quality: runs-on: ubuntu-latest needs: test @@ -55,7 +90,7 @@ jobs: build: runs-on: ubuntu-latest - needs: quality + needs: [quality, conformance] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 0000000..7957961 --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,55 @@ +name: release-gate + +on: + workflow_dispatch: + release: + types: [published, prereleased] + +permissions: + contents: read + +jobs: + gate: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev,image,pdf]" + python -m pip install build twine + + - name: Quality gates + run: | + pytest -q + ruff check . + mypy --strict aztec_py + + - name: Conformance evidence + run: | + python scripts/conformance_report.py \ + --report conformance_report.md \ + --json conformance_report.json \ + --matrix-report compat_matrix_report.md + + - name: Build and verify artifacts + run: | + python -m build + twine check dist/* + + - name: Upload release gate artifacts + uses: actions/upload-artifact@v4 + with: + name: release-gate-evidence + path: | + conformance_report.md + conformance_report.json + compat_matrix_report.md + dist/* diff --git a/.gitignore b/.gitignore index 72364f9..21e63e6 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,9 @@ htmlcov/ .coverage .coverage.* .cache +compat_matrix_report.md +conformance_report.md +conformance_report.json nosetests.xml coverage.xml *,cover diff --git a/MANIFEST.in b/MANIFEST.in index a3ada18..6e80058 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include requirements.txt include LICENSE include LICENSE.upstream include CONTRIBUTORS.md +include docs/ISO_IEC_24778_TRACEABILITY.md diff --git a/PRODUCTION_CHECKLIST.md b/PRODUCTION_CHECKLIST.md index e6202a2..9dcfda1 100644 --- a/PRODUCTION_CHECKLIST.md +++ b/PRODUCTION_CHECKLIST.md @@ -9,7 +9,9 @@ Use this checklist before shipping a new `aztec-py` version to production. - [ ] `python -m mypy --strict aztec_py` - [ ] `python -m build` - [ ] `python scripts/decoder_matrix.py --report compat_matrix_report.md` +- [ ] `python scripts/conformance_report.py --report conformance_report.md --json conformance_report.json --matrix-report compat_matrix_report.md` - [ ] If decode runtime is available in CI: `python scripts/decoder_matrix.py --strict-decode` +- [ ] `docs/ISO_IEC_24778_TRACEABILITY.md` reviewed and current ## 2. Runtime Optional Dependencies @@ -42,5 +44,6 @@ Use this checklist before shipping a new `aztec-py` version to production. ## 6. Incident Guardrails - [ ] Keep compatibility fixture failures as release blockers. +- [ ] Keep conformance report failures as release blockers. - [ ] Log scanner model/runtime for each production decode issue. - [ ] Add a regression fixture for every production bug before patching. diff --git a/README.md b/README.md index c42417e..8b5d37d 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,17 @@ Use strict mode when decode checks are mandatory in CI: python scripts/decoder_matrix.py --strict-decode ``` +Generate full conformance evidence (markdown + JSON + compatibility matrix): + +```bash +python scripts/conformance_report.py \ + --report conformance_report.md \ + --json conformance_report.json \ + --matrix-report compat_matrix_report.md +``` + Fixture source: `tests/compat/fixtures.json` +Traceability matrix: `docs/ISO_IEC_24778_TRACEABILITY.md` Release checklist: `PRODUCTION_CHECKLIST.md` ## CLI diff --git a/docs/ISO_IEC_24778_TRACEABILITY.md b/docs/ISO_IEC_24778_TRACEABILITY.md new file mode 100644 index 0000000..5dc7c1c --- /dev/null +++ b/docs/ISO_IEC_24778_TRACEABILITY.md @@ -0,0 +1,41 @@ +# ISO/IEC 24778 Traceability Matrix + +This document provides implementation traceability evidence for `aztec-py`. +It is intended for audit support and release validation workflows. + +This matrix does not replace independent certification. + +## Scope + +- Encoder implementation: `aztec_py/core.py` +- Symbol rendering: `aztec_py/renderers/*` +- Validation fixtures and regression checks: `tests/*`, `tests/compat/fixtures.json` + +## Traceability Table + +| Requirement Area | Implementation Evidence | Automated Verification | +|---|---|---| +| Symbol layer/size selection and capacity fit checks | `aztec_py/core.py` (`find_suitable_matrix_size`, `_required_capacity_bits`) | `tests/test_core.py`, `tests/test_validation.py` | +| Reed-Solomon error correction generation | `aztec_py/core.py` (`reed_solomon`) | `tests/test_core.py::Test::test_reed_solomon` | +| Character mode/latch/shift sequencing | `aztec_py/core.py` (`find_optimal_sequence`, `optimal_sequence_to_bits`) | `tests/test_core.py::Test::test_find_optimal_sequence_*`, `tests/test_core.py::Test::test_optimal_sequence_to_bits` | +| Bit stuffing and codeword construction | `aztec_py/core.py` (`get_data_codewords`) | `tests/test_core.py::Test::test_get_data_codewords` | +| CRLF handling regression | `aztec_py/core.py` + fixture/test coverage | `tests/test_core.py::Test::test_crlf_encoding`, `tests/test_core.py::Test::test_crlf_roundtrip` | +| Error-correction capacity regression (worst-case bytes) | `aztec_py/core.py` capacity calculations | `tests/test_core.py::Test::test_ec_worst_case_ff_bytes`, `tests/test_core.py::Test::test_ec_worst_case_null_bytes` | +| GS1 payload composition and separators | `aztec_py/gs1.py` | `tests/test_gs1.py` | +| Rendering determinism (PNG/SVG/PDF) | `aztec_py/core.py`, `aztec_py/renderers/image.py`, `aztec_py/renderers/svg.py` | `tests/test_renderers.py`, `tests/test_api_behaviour.py` | +| CLI behavior and output contract | `aztec_py/__main__.py` | `tests/test_cli.py` | +| Compatibility fixture corpus and decode matrix | `scripts/decoder_matrix.py`, `tests/compat/fixtures.json` | `tests/test_compat_matrix.py`, `scripts/conformance_report.py` | + +## Release Evidence Artifacts + +The following artifacts are generated and retained by CI/release gates: + +- `compat_matrix_report.md` +- `conformance_report.md` +- `conformance_report.json` + +## Audit Notes + +- Decode checks are runtime-dependent (`python-zxing` + Java). +- Non-strict mode allows skip-safe evidence generation when decode backend is unavailable. +- Strict mode can be enabled for environments where decode runtime is mandatory. diff --git a/scripts/conformance_report.py b/scripts/conformance_report.py new file mode 100644 index 0000000..c64cf9a --- /dev/null +++ b/scripts/conformance_report.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Generate audit-ready conformance evidence for aztec-py.""" + +from __future__ import annotations + +import argparse +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +import sys +from tempfile import NamedTemporaryFile + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from aztec_py import AztecCode # noqa: E402 +from aztec_py.compat import CompatCase, load_compat_cases # noqa: E402 +from aztec_py.decode import decode as decode_symbol # noqa: E402 + + +DEFAULT_FIXTURES = Path("tests/compat/fixtures.json") +DEFAULT_TRACEABILITY = Path("docs/ISO_IEC_24778_TRACEABILITY.md") + + +@dataclass(frozen=True) +class CaseOutcome: + """Result row for a compatibility case.""" + + case: str + payload: str + encode: str + decode: str + note: str + + +def _preview(payload: str | bytes) -> str: + if isinstance(payload, bytes): + return f"bytes[{len(payload)}]" + preview = payload.replace("\x1d", "") + if len(preview) > 48: + preview = f"{preview[:45]}..." + return preview + + +def _decode_backend_unavailable(message: str) -> bool: + lower = message.lower() + return ( + "optional dependency 'zxing'" in lower + or "java runtime" in lower + or "java" in lower and "failed to decode" in lower + ) + + +def _matches_expected(decoded: object, expected: str | bytes) -> bool: + if decoded == expected: + return True + + if isinstance(expected, str) and isinstance(decoded, bytes): + try: + return decoded.decode("utf-8") == expected + except UnicodeDecodeError: + return False + + if isinstance(expected, bytes) and isinstance(decoded, str): + try: + return decoded.encode("iso-8859-1") == expected + except UnicodeEncodeError: + return False + + return False + + +def _git_sha() -> str: + try: + return subprocess.check_output( + ["git", "rev-parse", "--short", "HEAD"], + cwd=REPO_ROOT, + text=True, + ).strip() + except Exception: # pragma: no cover - best effort + return "unknown" + + +def _run_case(case: CompatCase, module_size: int, strict_decode: bool) -> tuple[bool, CaseOutcome]: + try: + with NamedTemporaryFile(suffix=".png") as image_file: + AztecCode(case.payload, ec_percent=case.ec_percent, charset=case.charset).save( + image_file.name, + module_size=module_size, + ) + + if not case.decode_expected: + return True, CaseOutcome( + case=case.case_id, + payload=_preview(case.payload), + encode="pass", + decode="skip", + note="decode not required", + ) + + try: + decoded = decode_symbol(image_file.name) + except RuntimeError as exc: + message = str(exc) + if _decode_backend_unavailable(message): + if strict_decode: + return False, CaseOutcome( + case=case.case_id, + payload=_preview(case.payload), + encode="pass", + decode="fail", + note=f"decode backend unavailable: {message}", + ) + return True, CaseOutcome( + case=case.case_id, + payload=_preview(case.payload), + encode="pass", + decode="skip", + note=f"decode backend unavailable: {message}", + ) + return False, CaseOutcome( + case=case.case_id, + payload=_preview(case.payload), + encode="pass", + decode="fail", + note=message, + ) + + if _matches_expected(decoded, case.payload): + return True, CaseOutcome( + case=case.case_id, + payload=_preview(case.payload), + encode="pass", + decode="pass", + note="decoded payload matches", + ) + + return False, CaseOutcome( + case=case.case_id, + payload=_preview(case.payload), + encode="pass", + decode="fail", + note=f"decode mismatch for payload type {type(case.payload).__name__}", + ) + except Exception as exc: # pragma: no cover - guard for script use + return False, CaseOutcome( + case=case.case_id, + payload=_preview(case.payload), + encode="fail", + decode="skip", + note=str(exc), + ) + + +def _run_decoder_matrix( + fixtures: Path, + module_size: int, + strict_decode: bool, + report_path: Path, +) -> tuple[int, str]: + command = [ + sys.executable, + "scripts/decoder_matrix.py", + "--fixtures", + str(fixtures), + "--module-size", + str(module_size), + "--report", + str(report_path), + ] + if strict_decode: + command.append("--strict-decode") + + result = subprocess.run( + command, + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + return 0, "decoder matrix generated" + + combined = (result.stdout + "\n" + result.stderr).strip().splitlines() + tail = combined[-3:] if combined else ["decoder matrix failed with no output"] + return result.returncode, " | ".join(tail) + + +def _markdown_table_row(outcome: CaseOutcome) -> str: + return "| {case} | {payload} | {encode} | {decode} | {note} |".format( + case=outcome.case, + payload=outcome.payload.replace("|", "\\|"), + encode=outcome.encode, + decode=outcome.decode, + note=outcome.note.replace("|", "\\|"), + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate conformance evidence report.") + parser.add_argument( + "--fixtures", + type=Path, + default=DEFAULT_FIXTURES, + help="Path to compatibility fixture JSON.", + ) + parser.add_argument( + "--module-size", + type=int, + default=5, + help="Module size for generated symbol images.", + ) + parser.add_argument( + "--strict-decode", + action="store_true", + help="Fail when decode backend is unavailable for decode-expected cases.", + ) + parser.add_argument( + "--traceability", + type=Path, + default=DEFAULT_TRACEABILITY, + help="Path to traceability matrix markdown.", + ) + parser.add_argument( + "--matrix-report", + type=Path, + default=Path("compat_matrix_report.md"), + help="Output path for compatibility matrix markdown.", + ) + parser.add_argument( + "--report", + type=Path, + default=Path("conformance_report.md"), + help="Output path for conformance markdown report.", + ) + parser.add_argument( + "--json", + type=Path, + default=Path("conformance_report.json"), + help="Output path for machine-readable conformance JSON.", + ) + args = parser.parse_args() + + cases = load_compat_cases(args.fixtures) + outcomes: list[CaseOutcome] = [] + all_cases_ok = True + for case in cases: + case_ok, outcome = _run_case(case, module_size=args.module_size, strict_decode=args.strict_decode) + all_cases_ok = all_cases_ok and case_ok + outcomes.append(outcome) + + matrix_exit, matrix_note = _run_decoder_matrix( + fixtures=args.fixtures, + module_size=args.module_size, + strict_decode=args.strict_decode, + report_path=args.matrix_report, + ) + traceability_present = args.traceability.exists() + + encode_failures = sum(1 for row in outcomes if row.encode == "fail") + decode_failures = sum(1 for row in outcomes if row.decode == "fail") + decode_skips = sum(1 for row in outcomes if row.decode == "skip") + overall_ok = all_cases_ok and matrix_exit == 0 and traceability_present + + generated_at = datetime.now(timezone.utc).isoformat() + sha = _git_sha() + + markdown_lines = [ + "# aztec-py Conformance Report", + "", + f"- Generated (UTC): `{generated_at}`", + f"- Git SHA: `{sha}`", + f"- Fixtures: `{args.fixtures}` ({len(outcomes)} cases)", + f"- Strict decode mode: `{args.strict_decode}`", + "", + "## Control Status", + "", + f"- Traceability matrix: `{args.traceability}` ({'present' if traceability_present else 'missing'})", + f"- Compatibility matrix: `{args.matrix_report}` (exit code `{matrix_exit}`)", + f"- Compatibility matrix note: {matrix_note}", + "", + "## Summary", + "", + f"- Overall verdict: **{'PASS' if overall_ok else 'FAIL'}**", + f"- Cases total: `{len(outcomes)}`", + f"- Encode failures: `{encode_failures}`", + f"- Decode failures: `{decode_failures}`", + f"- Decode skips: `{decode_skips}`", + "", + "## Case Results", + "", + "| Case | Payload | Encode | Decode | Notes |", + "|---|---|---|---|---|", + ] + markdown_lines.extend(_markdown_table_row(row) for row in outcomes) + markdown_lines.append("") + + args.report.write_text("\n".join(markdown_lines), encoding="utf-8") + + json_payload = { + "generated_at_utc": generated_at, + "git_sha": sha, + "strict_decode": args.strict_decode, + "fixtures": str(args.fixtures), + "traceability": { + "path": str(args.traceability), + "present": traceability_present, + }, + "compat_matrix": { + "path": str(args.matrix_report), + "exit_code": matrix_exit, + "note": matrix_note, + }, + "summary": { + "overall_pass": overall_ok, + "total_cases": len(outcomes), + "encode_failures": encode_failures, + "decode_failures": decode_failures, + "decode_skips": decode_skips, + }, + "cases": [asdict(row) for row in outcomes], + } + args.json.write_text(json.dumps(json_payload, indent=2) + "\n", encoding="utf-8") + + print(f"Conformance report written to {args.report}") + print(f"Conformance JSON written to {args.json}") + print(f"Compatibility matrix written to {args.matrix_report}") + + return 0 if overall_ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_conformance_report.py b/tests/test_conformance_report.py new file mode 100644 index 0000000..3ef8fb2 --- /dev/null +++ b/tests/test_conformance_report.py @@ -0,0 +1,41 @@ +"""Smoke tests for conformance report generation.""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys + + +def test_conformance_report_script_smoke(tmp_path: Path) -> None: + report_path = tmp_path / "conformance_report.md" + json_path = tmp_path / "conformance_report.json" + matrix_path = tmp_path / "compat_matrix_report.md" + + result = subprocess.run( + [ + sys.executable, + "scripts/conformance_report.py", + "--report", + str(report_path), + "--json", + str(json_path), + "--matrix-report", + str(matrix_path), + ], + check=False, + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert report_path.exists() + assert json_path.exists() + assert matrix_path.exists() + + payload = json.loads(json_path.read_text(encoding="utf-8")) + assert payload["summary"]["overall_pass"] is True + assert payload["summary"]["total_cases"] > 0 + assert payload["traceability"]["present"] is True