diff --git a/.github/workflows/pr-orchestrator.yml b/.github/workflows/pr-orchestrator.yml index b72a8073..83b5486e 100644 --- a/.github/workflows/pr-orchestrator.yml +++ b/.github/workflows/pr-orchestrator.yml @@ -135,7 +135,7 @@ jobs: fi fi - - name: Run contract-first tests (no coverage) + - name: Run full test suite (smart-test-full) if: needs.changes.outputs.skip_tests_dev_to_main != 'true' shell: bash env: @@ -145,40 +145,23 @@ jobs: SMART_TEST_TIMEOUT_SECONDS: "1800" PYTEST_ADDOPTS: "-r fEw" run: | - echo "πŸ§ͺ Running contract-first test suite (3.12)..." + echo "πŸ§ͺ Running full test suite (smart-test-full, Python 3.12)..." echo "ℹ️ HATCH_TEST_ENV=${HATCH_TEST_ENV}" - run_layer() { - local label="$1" - shift - local start_ts - start_ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - echo "▢️ [${start_ts}] Starting ${label}" - echo " Command: $*" - if "$@"; then - local end_ts - end_ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - echo "βœ… [${end_ts}] ${label} completed" - else - local end_ts - end_ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - echo "⚠️ [${end_ts}] ${label} incomplete" - fi - } - - run_layer "Contract validation" hatch run contract-test-contracts - run_layer "Contract exploration" hatch run contract-test-exploration - run_layer "Scenario tests" hatch run contract-test-scenarios - run_layer "E2E tests" hatch run contract-test-e2e + hatch run smart-test-full - - name: Run unit tests with coverage (3.12) + - name: Generate coverage XML for quality gates if: needs.changes.outputs.skip_tests_dev_to_main != 'true' && env.RUN_UNIT_COVERAGE == 'true' - env: - PYTEST_ADDOPTS: "-r fEw" run: | - echo "πŸ§ͺ Running unit tests with coverage (3.12)..." - hatch -e hatch-test.py3.12 run run-cov hatch -e hatch-test.py3.12 run xml + - name: Upload test logs + if: needs.changes.outputs.skip_tests_dev_to_main != 'true' + uses: actions/upload-artifact@v4 + with: + name: test-logs + path: logs/tests/ + if-no-files-found: ignore + - name: Upload coverage artifacts if: needs.changes.outputs.skip_tests_dev_to_main != 'true' && env.RUN_UNIT_COVERAGE == 'true' uses: actions/upload-artifact@v4 @@ -220,10 +203,19 @@ jobs: - name: Run Python 3.11 compatibility tests (hatch-test matrix env) run: | echo "πŸ” Python 3.11 compatibility checks" - # Run a subset of tests to verify Python 3.11 compatibility - # Focus on unit tests and integration tests (skip slow E2E tests) - hatch -e hatch-test.py3.11 test -- -r fEw tests/unit tests/integration || echo "⚠️ Some tests failed (advisory)" - hatch -e hatch-test.py3.11 run xml || true + mkdir -p logs/compat-py311 + COMPAT_LOG="logs/compat-py311/compat_$(date -u +%Y%m%d_%H%M%S).log" + { + hatch -e hatch-test.py3.11 run run -- -r fEw tests/unit tests/integration || echo "⚠️ Some tests failed (advisory)" + hatch -e hatch-test.py3.11 run xml || true + } 2>&1 | tee "$COMPAT_LOG" + - name: Upload compat-py311 logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: compat-py311-logs + path: logs/compat-py311/ + if-no-files-found: ignore contract-first-ci: name: Contract-First CI @@ -255,10 +247,29 @@ jobs: restore-keys: | ${{ runner.os }}-hatch-contract-first-py312- ${{ runner.os }}-hatch- + - name: Prepare repro log directory + run: mkdir -p logs/repro - name: Run contract validation and exploration + id: repro run: | echo "πŸ” Validating runtime contracts..." - echo "Running specfact repro with required CrossHair..." && hatch run specfact repro --verbose --crosshair-required --budget 120 || echo "SpecFact repro found issues" + REPRO_LOG="logs/repro/repro_$(date -u +%Y%m%d_%H%M%S).log" + echo "Running specfact repro with required CrossHair... (log: $REPRO_LOG)" + hatch run specfact repro --verbose --crosshair-required --budget 120 2>&1 | tee "$REPRO_LOG" || echo "SpecFact repro found issues" + - name: Upload repro logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: repro-logs + path: logs/repro/ + if-no-files-found: ignore + - name: Upload repro reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: repro-reports + path: .specfact/reports/enforcement/ + if-no-files-found: ignore cli-validation: name: CLI Command Validation @@ -313,19 +324,30 @@ jobs: echo "βœ… Found coverage.xml" - name: Run quality gates (advisory) run: | - echo "πŸ” Checking coverage (advisory only)..." - COVERAGE_PERCENT_XML=$(grep -o "line-rate=\"[0-9.]*\"" logs/tests/coverage/coverage.xml | head -1 | sed 's/line-rate=\"//' | sed 's/\"//') - if [ -n "$COVERAGE_PERCENT_XML" ] && [ "$COVERAGE_PERCENT_XML" != "0" ]; then - COVERAGE_PERCENT_INT=$(echo "$COVERAGE_PERCENT_XML * 100" | bc -l | cut -d. -f1) - else - COVERAGE_PERCENT_INT=0 - fi - echo "πŸ“Š Line coverage (advisory): ${COVERAGE_PERCENT_INT}%" - if [ "$COVERAGE_PERCENT_INT" -lt 30 ]; then - echo "⚠️ Advisory: coverage below 30% β€” permitted under contract-first; prioritize contract/scenario gaps." - else - echo "βœ… Advisory: coverage acceptable" - fi + mkdir -p logs/quality-gates + QG_LOG="logs/quality-gates/quality-gates_$(date -u +%Y%m%d_%H%M%S).log" + { + echo "πŸ” Checking coverage (advisory only)..." + COVERAGE_PERCENT_XML=$(grep -o "line-rate=\"[0-9.]*\"" logs/tests/coverage/coverage.xml | head -1 | sed 's/line-rate=\"//' | sed 's/\"//') + if [ -n "$COVERAGE_PERCENT_XML" ] && [ "$COVERAGE_PERCENT_XML" != "0" ]; then + COVERAGE_PERCENT_INT=$(echo "$COVERAGE_PERCENT_XML * 100" | bc -l | cut -d. -f1) + else + COVERAGE_PERCENT_INT=0 + fi + echo "πŸ“Š Line coverage (advisory): ${COVERAGE_PERCENT_INT}%" + if [ "$COVERAGE_PERCENT_INT" -lt 30 ]; then + echo "⚠️ Advisory: coverage below 30% β€” permitted under contract-first; prioritize contract/scenario gaps." + else + echo "βœ… Advisory: coverage acceptable" + fi + } 2>&1 | tee "$QG_LOG" + - name: Upload quality-gates logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: quality-gates-logs + path: logs/quality-gates/ + if-no-files-found: ignore type-checking: name: Type Checking (basedpyright) @@ -360,8 +382,17 @@ jobs: - name: Run type checking run: | echo "πŸ” Running basedpyright type checking..." - # Fail on type errors (severity 8) to enforce type safety in CI/CD - hatch run type-check + mkdir -p logs/type-check + TYPE_CHECK_LOG="logs/type-check/type-check_$(date -u +%Y%m%d_%H%M%S).log" + hatch run type-check 2>&1 | tee "$TYPE_CHECK_LOG" + exit "${PIPESTATUS[0]:-$?}" + - name: Upload type-check logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: type-check-logs + path: logs/type-check/ + if-no-files-found: ignore linting: name: Linting (ruff, pylint) @@ -401,7 +432,16 @@ jobs: - name: Run linting run: | echo "πŸ” Running linting checks..." - hatch run lint || echo "⚠️ Linting incomplete" + mkdir -p logs/lint + LINT_LOG="logs/lint/lint_$(date -u +%Y%m%d_%H%M%S).log" + hatch run lint 2>&1 | tee "$LINT_LOG" || echo "⚠️ Linting incomplete" + - name: Upload lint logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: lint-logs + path: logs/lint/ + if-no-files-found: ignore package-validation: name: Package Validation (uvx/pip) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70032795..c96f3b13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,20 @@ All notable changes to this project will be documented in this file. --- +## [0.31.1] - 2026-02-16 + +### Added + +- CI log artifacts: PR Orchestrator workflow now uploads test logs (`test-logs`) from `hatch run smart-test-full` and repro logs/reports (`repro-logs`, `repro-reports`) from the contract-first-ci job so failed runs can be debugged by downloading full logs from the Actions Artifacts section without re-running locally. +- Documentation: "CI and GitHub Actions" section in [Troubleshooting](docs/guides/troubleshooting.md) describing artifact names and how to download and use them. + +### Changed + +- Tests job in `.github/workflows/pr-orchestrator.yml` now runs `hatch run smart-test-full` (single full-suite step with log output to `logs/tests/`) and uploads `logs/tests/` as the `test-logs` artifact. +- Contract-first-ci job captures `specfact repro` stdout/stderr to `logs/repro/` and uploads `repro-logs` and `repro-reports` (`.specfact/reports/enforcement/`) as artifacts on every run. + +--- + ## [0.31.0] - 2026-02-13 ### Added diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index d0ab8d09..f4033f4e 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -789,6 +789,38 @@ The command automatically uses tokens in this order: --- +## CI and GitHub Actions + +### Downloading test and repro logs from CI + +When a PR or push runs the **PR Orchestrator** workflow, test and repro output are uploaded as workflow artifacts so you can debug failures without re-running the full suite locally. + +1. **Where to find artifacts** + + - Open the workflow run: **Actions** β†’ select the **PR Orchestrator** run (e.g. from your PR or branch). + - Scroll to the **Artifacts** section at the bottom of the run. + +2. **Artifact names and contents** + + | Artifact | Job | Contents | + |----------------------|------------------|---------------------------------------------------------------------------| + | `test-logs` | Tests (Python 3.12) | Full test run and coverage logs from `hatch run smart-test-full` (`logs/tests/`). | + | `coverage-reports` | Tests (Python 3.12) | Coverage XML for quality gates (when unit tests ran). | + | `compat-py311-logs` | Compatibility (Python 3.11) | Pytest and coverage XML output from the compat job. | + | `type-check-logs` | Type Checking (basedpyright) | Full basedpyright type-check output. | + | `lint-logs` | Linting (ruff, pylint) | Full lint run output. | + | `quality-gates-logs`| Quality Gates (Advisory) | Coverage percentage and advisory message. | + | `repro-logs` | Contract-First CI | Full stdout/stderr of `specfact repro` (`logs/repro/`). | + | `repro-reports` | Contract-First CI | Repro report YAMLs from `.specfact/reports/enforcement/`. | + +3. **How to use them** + + - Download the artifact (e.g. `test-logs` or `repro-logs`) from the run page. + - Unzip and open the log or report files to see the full output that led to the failure. + - Use this instead of copying snippets from the step log, so you get complete error context before fixing and pushing again. + +--- + ## Getting Help If you're still experiencing issues: diff --git a/docs/plans/ci-pr-orchestrator-log-artifacts.md b/docs/plans/ci-pr-orchestrator-log-artifacts.md new file mode 100644 index 00000000..069a260b --- /dev/null +++ b/docs/plans/ci-pr-orchestrator-log-artifacts.md @@ -0,0 +1,44 @@ +# Plan: PR Orchestrator β€” Attach Test and Repro Logs to CI Runs + +**Repository**: `nold-ai/specfact-cli` (public) + +## Purpose + +Improve the GitHub Action runner (`.github/workflows/pr-orchestrator.yml`) so that: + +1. **Full test output is available on failure** β€” Today we often copy-paste from the UI, error details are truncated or only snippets, and we must re-run locally to find all issues. The goal is to run `smart-test-full` (or equivalent) so that log files are generated and attached to each CI run so we can download them when a run fails. + +2. **Repro test logs are captured and attached** β€” For the `specfact repro` step (contract-first-ci job), collect logs and the repro report directory (e.g. `.specfact/reports/enforcement`) and upload them as artifacts so we can download on error. + +3. **Shift full execution validation to CI** β€” By having full logs and repro artifacts attached, we can rely on CI for full validation before merging to `dev` and avoid redundant local full runs. + +## Current Shortcomings + +- **pr-orchestrator.yml** runs contract-first test layers (contract-test-contracts, contract-test-exploration, contract-test-scenarios, contract-test-e2e) but does **not** use `smart-test-full`; it does not write test output to log files that are then uploaded. +- Only **coverage.xml** is uploaded (in Tests job); no raw test logs or repro logs. +- The **contract-first-ci** job runs `hatch run specfact repro --verbose --crosshair-required --budget 120` with `|| echo "SpecFact repro found issues"`, so the job does not fail hard and repro stdout/stderr and reports are not uploaded as artifacts. +- Error details in the GitHub Actions UI are limited to step output (snippets); full logs are not downloadable. + +## What Changes + +- **Tests job**: Switch to (or add) running `hatch run smart-test-full` so that the smart-test script writes logs under `logs/tests/` (e.g. `full_test_run_*.log`, `full_coverage_*.log`). Upload the contents of `logs/tests/` (and existing `logs/tests/coverage/coverage.xml`) as workflow artifacts so they are available for download on every run (or on failure only to save space/retention). +- **Contract-first-ci job (repro)**: After running `specfact repro`, collect (1) repro stdout/stderr (e.g. by redirecting to a file or using a wrapper that writes to `logs/repro/`), and (2) `.specfact/reports/enforcement/` (repro reports). Upload these as artifacts (e.g. `repro-logs` and `repro-reports`), so on failure we can download full repro output and reports. +- **Artifact upload strategy**: Use `actions/upload-artifact@v4` with a consistent naming scheme (e.g. `test-logs-`, `repro-logs`, `repro-reports`). Optionally use `if: failure()` or `if: always()` so artifacts are retained for failed runs or all runs per project policy. +- **Documentation**: Update docs (e.g. contributing, troubleshooting, or reference) to mention that CI produces downloadable test and repro log artifacts and how to use them. + +## Files to Create/Modify + +- `.github/workflows/pr-orchestrator.yml` β€” Add/change test step to use smart-test-full with log output; add artifact upload steps for test logs and repro logs/reports. +- Optionally: a small script under `.github/workflows/scripts/` to run repro and capture logs to a file (if not done inline). +- `docs/` β€” Add or update a section on CI artifacts (where to find them, what they contain). + +## Success Metrics + +- Every PR/push run that executes tests produces downloadable artifacts containing full test log files when using smart-test-full. +- Every run that executes `specfact repro` produces downloadable artifacts containing repro stdout/stderr and repro report files. +- Contributors can diagnose failures from downloaded artifacts without needing to re-run the full suite locally. + +## Dependencies + +- Existing `tools/smart_test_coverage.py` already writes to `logs/tests/` when running full tests; ensure CI has write access and paths are consistent. +- Existing `specfact.yml` already uploads `.specfact/reports/enforcement/*.yaml` and `.specfact/pr-comment.md`; align naming and behavior with pr-orchestrator for repro artifacts. diff --git a/openspec/CHANGE_ORDER.md b/openspec/CHANGE_ORDER.md index ee036f49..ebc04061 100644 --- a/openspec/CHANGE_ORDER.md +++ b/openspec/CHANGE_ORDER.md @@ -65,6 +65,12 @@ These are derived extensions of the same 2026-02-15 plan and are required to ope | validation | 01 | validation-01-deep-validation | [#163](https://github.com/nold-ai/specfact-cli/issues/163) | β€” | | bundle-mapper | 01 | bundle-mapper-01-mapping-strategy | [#121](https://github.com/nold-ai/specfact-cli/issues/121) | β€” | +### CI/CD (workflow and artifacts) + +| Module | Order | Change folder | GitHub # | Blocked by | +|--------|-------|----------------|----------|------------| +| ci | 01 | ci-01-pr-orchestrator-log-artifacts | [#260](https://github.com/nold-ai/specfact-cli/issues/260) | β€” | + ### backlog-core (required by all backlog-* modules) | Module | Order | Change folder | GitHub # | Blocked by | diff --git a/openspec/changes/ci-01-pr-orchestrator-log-artifacts/.openspec.yaml b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/.openspec.yaml new file mode 100644 index 00000000..a5571c18 --- /dev/null +++ b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-16 diff --git a/openspec/changes/ci-01-pr-orchestrator-log-artifacts/CHANGE_VALIDATION.md b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/CHANGE_VALIDATION.md new file mode 100644 index 00000000..2e92f3cf --- /dev/null +++ b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/CHANGE_VALIDATION.md @@ -0,0 +1,61 @@ +# Change Validation Report: ci-01-pr-orchestrator-log-artifacts + +**Validation Date**: 2026-02-16 +**Change Proposal**: [proposal.md](./proposal.md) +**Validation Method**: Dry-run simulation; format and OpenSpec strict validation + +## Executive Summary + +- Breaking Changes: 0 detected +- Dependent Files: 0 (workflow and docs only; no application code interfaces changed) +- Impact Level: Low +- Validation Result: Pass +- User Decision: N/A (no breaking changes) + +## Breaking Changes Detected + +None. This change only modifies `.github/workflows/pr-orchestrator.yml` (add/change steps for smart-test-full and artifact uploads) and adds or updates documentation. No public API, contract, or application code changes. + +## Dependencies Affected + +- **Workflow**: `.github/workflows/pr-orchestrator.yml` β€” add steps; existing jobs (tests, contract-first-ci) extended with new steps. No other workflows depend on pr-orchestrator's internal step names. +- **Docs**: One new or updated section (troubleshooting or contributing). No code imports or references to workflow artifact names outside docs. + +## Impact Assessment + +- **Code Impact**: None (workflow YAML and docs only). +- **Test Impact**: None; optional manual verification that artifacts appear in a CI run. +- **Documentation Impact**: New or updated subsection on CI artifacts (where to find them, what they contain). +- **Release Impact**: Patch (CI improvement, no user-facing API change). + +## Format Validation + +- **proposal.md Format**: Pass + - Title: `# Change: CI β€” Attach Test and Repro Log Artifacts...` (correct). + - Required sections: Why, What Changes, Capabilities, Impact, Source Tracking present. + - Capabilities section: One capability `ci-log-artifacts` with spec file `specs/ci-log-artifacts/spec.md`. +- **tasks.md Format**: Pass + - TDD/SDD order section at top; hierarchical numbered tasks; branch creation first (## 1), PR creation last (## 9). + - Sub-tasks use `- [ ] N.M.N` format. +- **specs Format**: Pass + - `specs/ci-log-artifacts/spec.md` uses Given/When/Then; ADDED requirements with scenarios. +- **design.md Format**: Pass + - Overview, current/target state, integration points, edge cases; no bridge adapters (N/A). +- **Config.yaml Compliance**: Pass + - Git workflow: branch first, PR last; quality gates; documentation task; version/changelog before PR. + - No GitHub issue creation task in tasks.md; proposal has Source Tracking placeholder for issue (to be created). + +## OpenSpec Validation + +- **Status**: Pass +- **Validation Command**: `openspec validate ci-01-pr-orchestrator-log-artifacts --strict` +- **Issues Found**: 0 +- **Issues Fixed**: 0 +- **Re-validated**: N/A + +## Next Steps + +1. Create GitHub issue in nold-ai/specfact-cli (title: `[Change] Attach test and repro log artifacts to PR orchestrator runs`; labels: enhancement, change-proposal). +2. Update proposal.md Source Tracking with issue number and URL. +3. Proceed with implementation: `/opsx:apply ci-01-pr-orchestrator-log-artifacts` or apply tasks manually. +4. Update CHANGE_ORDER.md when change is created (add row under a CI or cross-cutting section). diff --git a/openspec/changes/ci-01-pr-orchestrator-log-artifacts/design.md b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/design.md new file mode 100644 index 00000000..64278302 --- /dev/null +++ b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/design.md @@ -0,0 +1,33 @@ +# Design: CI Log Artifacts for PR Orchestrator + +## Overview + +This change updates `.github/workflows/pr-orchestrator.yml` so that (1) the Tests job runs `hatch run smart-test-full` and uploads generated logs under `logs/tests/` as artifacts, and (2) the contract-first-ci job captures `specfact repro` output to a log file and uploads repro logs and `.specfact/reports/enforcement/` as artifacts. No new external systems; only workflow YAML and optional helper scripts. + +## Current State + +- **Tests job**: Runs contract-test layers (contract-test-contracts, contract-test-exploration, contract-test-scenarios, contract-test-e2e) and unit tests with coverage via `hatch -e hatch-test.py3.12 run run-cov` / `hatch -e hatch-test.py3.12 run xml`. Uploads only `logs/tests/coverage/coverage.xml` as `coverage-reports`. No full test stdout/stderr log files are produced or uploaded. +- **contract-first-ci job**: Runs `hatch run specfact repro --verbose --crosshair-required --budget 120` with `|| echo "SpecFact repro found issues"`. No log file capture; no artifact upload for repro output or reports. +- **smart_test_coverage.py**: When run with `--level full`, writes to `logs/tests/` (e.g. `full_test_run_.log`, `full_coverage_.log`). Used locally; not yet used in pr-orchestrator. + +## Target State + +- **Tests job**: Add or replace a step to run `hatch run smart-test-full` so that logs are written to `logs/tests/`. Keep or align with existing coverage XML generation so quality-gates job still receives coverage. Add an upload-artifact step that uploads `logs/tests/` (and `logs/tests/coverage/` if separate) with a name like `test-logs`. Use `if: always()` so logs are available for both success and failure. +- **contract-first-ci job**: Before or as part of the repro step, ensure `logs/repro/` exists. Run repro with stdout/stderr redirected to a file (e.g. `logs/repro/repro_$(date +%Y%m%d_%H%M%S).log`). After the repro step, add an upload-artifact step that uploads `logs/repro/` and `.specfact/reports/enforcement/` (if present) with names like `repro-logs` and `repro-reports`, with `if: always()`. +- **Artifact retention**: Rely on GitHub Actions default retention; no change to workflow-level retention unless required by org policy. + +## Integration Points + +- **smart_test_coverage.py**: Already creates `logs/tests/` and writes timestamped log files when running full tests. CI must run in an environment where `hatch run smart-test-full` is available (Python 3.12, hatch, dependencies). Timeout: existing step may use SMART_TEST_TIMEOUT_SECONDS (e.g. 1800); ensure sufficient for full run. +- **specfact repro**: Writes reports to `.specfact/reports/enforcement/` (existing). Repro does not write its own stdout to a file; workflow must redirect (e.g. `tee` or `script -q -c "..." repro.log`). +- **specfact.yml**: Already uploads `.specfact/reports/enforcement/*.yaml` and `.specfact/pr-comment.md` as `specfact-report`. We keep pr-orchestrator artifact names distinct (e.g. `repro-reports`) so both workflows remain consistent and downloadable. + +## Edge Cases + +- **devβ†’main skip**: When `skip_tests_dev_to_main` is true, Tests and contract-first-ci are skipped; no new artifact steps run. No change. +- **No report directory**: If `specfact repro` never creates `.specfact/reports/enforcement/`, upload with `if-no-files-found: ignore` so the step does not fail. +- **Log directory missing**: Create `logs/repro/` in the contract-first-ci job before running repro so the redirect target exists. + +## Contract Enforcement + +- No new Python public APIs; only workflow and docs. No new @icontract/@beartype. Existing tools (smart_test_coverage.py, specfact repro) are unchanged except how they are invoked from CI. diff --git a/openspec/changes/ci-01-pr-orchestrator-log-artifacts/proposal.md b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/proposal.md new file mode 100644 index 00000000..9d0490aa --- /dev/null +++ b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/proposal.md @@ -0,0 +1,30 @@ +# Change: CI β€” Attach Test and Repro Log Artifacts to PR Orchestrator Runs + +## Why + +The current GitHub Action runner (`.github/workflows/pr-orchestrator.yml`) forces us to copy-paste from the UI when debugging; error details are often truncated or only snippets are visible, so we must re-run the full suite locally to find all issues before fixing them. Repro test output and reports are not attached to the run, so failures in `specfact repro` are hard to diagnose from CI alone. The goal is to use `smart-test-full` to generate log files, attach those logs (and repro logs/reports) as workflow artifacts so we can download them on failure, and shift full execution validation from local runs to CI before merging to `dev`. + +## What Changes + +- **Tests job**: Use `hatch run smart-test-full` so that the smart-test script writes full test and coverage logs under `logs/tests/`. Upload `logs/tests/` (and existing coverage XML) as workflow artifacts so every run (or every failed run) has downloadable full logs. +- **Contract-first-ci job (repro)**: After running `specfact repro`, capture repro stdout/stderr to a file under `logs/repro/` and upload `.specfact/reports/enforcement/` and the repro log file as artifacts so we can download them when the job fails. +- **Artifact upload**: Add `actions/upload-artifact@v4` steps for test logs and repro logs/reports with consistent names (e.g. `test-logs`, `repro-logs`, `repro-reports`). Use `if: always()` or `if: failure()` per project policy so artifacts are available for debugging. +- **Documentation**: Update contributing or troubleshooting docs to describe CI log artifacts and how to download and use them. + +## Capabilities + +- **ci-log-artifacts**: PR orchestrator runs produce downloadable test logs (from smart-test-full) and repro logs/reports so failures can be diagnosed from CI without local re-runs. + +## Impact + +- **CI**: Longer test step when using smart-test-full (writes logs); additional artifact upload steps; slightly more storage/retention for artifacts. No change to test semantics beyond ensuring full logs are generated and uploaded. +- **Developers**: Can download full test and repro logs from the Actions run when a job fails, reducing need to re-run locally. +- **Docs**: One new or updated section on CI artifacts (where to find them, what they contain). + +## Source Tracking + + +- **GitHub Issue**: #260 +- **Issue URL**: +- **Repository**: nold-ai/specfact-cli +- **Last Synced Status**: proposed diff --git a/openspec/changes/ci-01-pr-orchestrator-log-artifacts/specs/ci-log-artifacts/spec.md b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/specs/ci-log-artifacts/spec.md new file mode 100644 index 00000000..345849ad --- /dev/null +++ b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/specs/ci-log-artifacts/spec.md @@ -0,0 +1,92 @@ +# CI Log Artifacts + +## ADDED Requirements + +### Requirement: Full Test Logs from Smart-Test-Full in CI + +The PR orchestrator workflow SHALL run the full test suite via `hatch run smart-test-full` (or equivalent) so that test and coverage logs are written under `logs/tests/`, and those logs SHALL be uploaded as workflow artifacts so they can be downloaded when a run fails. + +**Rationale**: Today only snippets appear in the GitHub UI; full logs are needed to diagnose failures without re-running locally. + +#### Scenario: Tests Job Produces and Uploads Test Logs + +**Given**: A PR or push that triggers the PR orchestrator and runs the Tests job (code changed, not devβ†’main skip) + +**When**: The Tests job runs `hatch run smart-test-full` (or a step that invokes the smart-test script with level `full`) + +**Then**: The smart-test script writes test run output and coverage output to files under `logs/tests/` (e.g. `full_test_run_.log`, `full_coverage_.log` or equivalent), and a subsequent step uploads the contents of `logs/tests/` (and any existing `logs/tests/coverage/coverage.xml`) as a workflow artifact (e.g. name `test-logs` or `test-logs-py312`) + +**Acceptance Criteria**: + +- Tests job runs full suite in a way that generates log files under `logs/tests/` +- Artifact upload step uses `actions/upload-artifact@v4` with path including `logs/tests/` +- Artifacts are available for download from the Actions run (on failure or always, per policy) + +#### Scenario: Download Test Logs After Failed Tests Job + +**Given**: The Tests job failed (e.g. smart-test-full exited non-zero) + +**When**: A developer opens the workflow run in GitHub Actions and goes to the Artifacts section + +**Then**: An artifact (e.g. `test-logs`) is present and contains at least one test log file and, when coverage was run, coverage XML or coverage log, so the developer can inspect full output without re-running locally + +**Acceptance Criteria**: + +- Failed runs that produced logs have a downloadable artifact with those logs +- Artifact naming is consistent so it can be referenced in docs + +--- + +### Requirement: Repro Logs and Reports Attached to CI Run + +The contract-first-ci job (which runs `specfact repro`) SHALL capture repro command stdout/stderr to a log file and SHALL upload that log file plus the repro report directory (e.g. `.specfact/reports/enforcement/`) as workflow artifacts so they can be downloaded when the job fails. + +**Rationale**: Repro failures are currently hard to diagnose from CI because only step output (truncated) is visible; full repro output and report YAMLs are needed. + +#### Scenario: Contract-First-CI Job Captures and Uploads Repro Logs + +**Given**: The contract-first-ci job runs `specfact repro --verbose --crosshair-required --budget 120` (or equivalent) + +**When**: The repro command runs (whether it passes or fails) + +**Then**: (1) Stdout and stderr of the repro command are captured to a file under `logs/repro/` (e.g. `repro_.log`), and (2) the contents of `.specfact/reports/enforcement/` (if present) are uploaded together with the repro log as workflow artifacts (e.g. `repro-logs` and `repro-reports` or a single `repro-artifacts` artifact) + +**Acceptance Criteria**: + +- Repro command output is written to a file in `logs/repro/` (directory created if needed) +- Upload step runs after repro (e.g. `if: always()`) so artifacts are available even when repro fails +- Artifact(s) include the repro log file and any report YAMLs under `.specfact/reports/enforcement/` + +#### Scenario: Download Repro Artifacts After Failed Repro Step + +**Given**: The contract-first-ci job ran and the repro step failed (or completed with issues) + +**When**: A developer opens the workflow run and goes to the Artifacts section + +**Then**: An artifact such as `repro-logs` or `repro-reports` is present and contains the full repro log and report files so the developer can diagnose without re-running `specfact repro` locally + +**Acceptance Criteria**: + +- Naming and path are documented so developers know where to find repro logs and reports +- Align with existing specfact.yml behavior (that workflow already uploads `.specfact/reports/enforcement/*.yaml`) for consistency + +--- + +### Requirement: Documentation for CI Log Artifacts + +The documentation SHALL describe where to find test and repro log artifacts in GitHub Actions and how to use them for debugging failed runs. + +**Rationale**: Contributors need to know that artifacts exist and what they contain. + +#### Scenario: Contributor Finds CI Artifact Documentation + +**Given**: A contributor has a failed CI run and wants to debug without re-running locally + +**When**: They look in the contributing guide, troubleshooting guide, or a reference section on CI + +**Then**: They find a short section explaining that test logs and repro logs/reports are uploaded as artifacts, how to download them from the Actions run (Artifacts section), and what each artifact contains (test output, coverage, repro stdout/stderr, repro report YAMLs) + +**Acceptance Criteria**: + +- At least one doc page (e.g. `docs/guides/troubleshooting.md` or `docs/contributing/`) includes a subsection on CI artifacts +- Section is copy-paste or link friendly (e.g. "Go to the run β†’ Artifacts β†’ download test-logs or repro-logs") diff --git a/openspec/changes/ci-01-pr-orchestrator-log-artifacts/tasks.md b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/tasks.md new file mode 100644 index 00000000..99e924b6 --- /dev/null +++ b/openspec/changes/ci-01-pr-orchestrator-log-artifacts/tasks.md @@ -0,0 +1,62 @@ +# Tasks: CI β€” Attach Test and Repro Log Artifacts to PR Orchestrator Runs + +## TDD / SDD order (enforced) + +Per `openspec/config.yaml`, **tests before code** apply to any task that adds or changes behavior. Order: (1) Spec deltas define behavior (Given/When/Then). (2) **Tests second** β€” write unit/integration tests from those scenarios; run tests and **expect failure** (no implementation yet). (3) **Code last** β€” implement until tests pass and behavior satisfies the spec. Do not implement production code for new behavior until the corresponding tests exist and have been run (expecting failure). + +For this change, the main deliverable is workflow YAML and docs; "tests" are satisfied by workflow lint (`hatch run lint-workflows`) and optional manual/e2e verification that artifacts appear. No new application code; TDD for this change is: validate workflow syntax and lint first, then implement workflow and upload steps, then verify artifacts in a run. + +--- + +## 1. Create git branch + +- [ ] 1.1 Ensure we're on dev and up to date: `git checkout dev && git pull origin dev` +- [ ] 1.2 Create branch: `gh issue develop --repo nold-ai/specfact-cli --name feature/ci-01-pr-orchestrator-log-artifacts --checkout` if issue exists, else `git checkout -b feature/ci-01-pr-orchestrator-log-artifacts` +- [ ] 1.3 Verify branch: `git branch --show-current` + +## 2. Verify spec deltas (SDD: specs first) + +- [ ] 2.1 Confirm `specs/ci-log-artifacts/spec.md` exists and is complete (Given/When/Then for test logs upload, repro logs/reports upload, documentation). +- [ ] 2.2 Map scenarios to implementation: Tests job smart-test-full + artifact upload; contract-first-ci repro log capture + artifact upload; doc section on CI artifacts. + +## 3. Tests job: Run smart-test-full and upload test logs (TDD: validate then implement) + +- [ ] 3.1 **Validation**: Run `hatch run lint-workflows` (or equivalent) to ensure workflow syntax is valid; note current pr-orchestrator.yml structure. +- [ ] 3.2 In `.github/workflows/pr-orchestrator.yml`, add or replace the test execution step in the **Tests** job so that it runs `hatch run smart-test-full` (with env such as `CONTRACT_FIRST_TESTING`, `TEST_MODE`, `HATCH_TEST_ENV`, `SMART_TEST_TIMEOUT_SECONDS`, `PYTEST_ADDOPTS` as needed). Ensure the script writes logs under `logs/tests/` (existing behavior of smart_test_coverage.py when level is full). +- [ ] 3.3 Add a step to upload test log artifacts: use `actions/upload-artifact@v4` with name `test-logs` (or `test-logs-py312`), path `logs/tests/`, and `if-no-files-found: ignore` or `warn` so the job does not fail if no logs (e.g. when step was skipped). Use `if: always()` or `if: success() || failure()` so artifacts are uploaded on both success and failure when the step ran. +- [ ] 3.4 Keep or adjust the existing "Upload coverage artifacts" step so quality-gates still receives coverage (e.g. continue uploading `logs/tests/coverage/coverage.xml` as `coverage-reports` if that path is still produced by smart-test-full or a separate coverage step). +- [ ] 3.5 Re-run `hatch run lint-workflows` and fix any issues. + +## 4. Contract-first-ci job: Capture repro output and upload repro logs/reports + +- [ ] 4.1 In the **contract-first-ci** job, ensure `logs/repro/` exists before running repro (e.g. `mkdir -p logs/repro`). +- [ ] 4.2 Change the repro run step so that stdout and stderr are captured to a timestamped file under `logs/repro/` (e.g. `repro_$(date -u +%Y%m%d_%H%M%S).log`) using `tee` or redirection, while still displaying output in the step log. Run `hatch run specfact repro --verbose --crosshair-required --budget 120`; keep `|| echo "SpecFact repro found issues"` or similar so the job can continue to upload artifacts even when repro fails. +- [ ] 4.3 Add an upload-artifact step for repro logs: upload `logs/repro/` with name `repro-logs`, `if-no-files-found: ignore`, and `if: always()` so it runs after the repro step whether repro passed or failed. +- [ ] 4.4 Add an upload-artifact step for repro reports: upload `.specfact/reports/enforcement/` with name `repro-reports`, `if-no-files-found: ignore`, and `if: always()`. +- [ ] 4.5 Run `hatch run lint-workflows` again. + +## 5. Documentation: CI log artifacts + +- [ ] 5.1 Identify the best doc location (e.g. `docs/guides/troubleshooting.md`, `docs/contributing/`, or a new `docs/reference/ci-artifacts.md`). Add or update a subsection that explains: (1) test logs and repro logs/reports are uploaded as workflow artifacts; (2) where to find them (Actions run β†’ Artifacts); (3) artifact names (`test-logs`, `repro-logs`, `repro-reports`) and what they contain; (4) how to use them to debug failed runs without re-running locally. +- [ ] 5.2 If adding a new page, set front-matter (layout, title, permalink, description) and update `docs/_layouts/default.html` sidebar if needed. + +## 6. Quality gates + +- [ ] 6.1 Run `hatch run format`, `hatch run type-check`. +- [ ] 6.2 Run `hatch run lint` and `hatch run yaml-lint`; run `hatch run lint-workflows` for workflow files. +- [ ] 6.3 Run `hatch run contract-test` and `hatch run smart-test` (or `smart-test-unit` / `smart-test-folder` for minimal validation). No new application code; ensure no regressions. + +## 7. Documentation research and review (per openspec/config.yaml) + +- [ ] 7.1 Confirm affected docs are listed in task 5; check for broken links and correct front-matter. + +## 8. Version and changelog (required before PR) + +- [ ] 8.1 Bump patch version (this is a fix/enhancement to CI): update `pyproject.toml`, `setup.py`, `src/__init__.py`, `src/specfact_cli/__init__.py`. +- [ ] 8.2 Add CHANGELOG.md entry under new version: Added β€” CI log artifacts (test logs and repro logs/reports) attached to PR orchestrator runs for easier debugging. + +## 9. Create Pull Request to dev + +- [ ] 9.1 Commit and push: `git add .`, `git commit -m "feat(ci): attach test and repro log artifacts to PR orchestrator runs"`, `git push origin feature/ci-01-pr-orchestrator-log-artifacts`. +- [ ] 9.2 Create PR: `gh pr create --repo nold-ai/specfact-cli --base dev --head feature/ci-01-pr-orchestrator-log-artifacts --title "feat(ci): attach test and repro log artifacts to PR orchestrator runs" --body-file ` (use PR template; reference OpenSpec change `ci-01-pr-orchestrator-log-artifacts` and link to GitHub issue if created). +- [ ] 9.3 Verify PR and branch are linked to the issue in the Development section. diff --git a/pyproject.toml b/pyproject.toml index f9a5b64a..efd8cdcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "specfact-cli" -version = "0.31.0" +version = "0.31.1" description = "The swiss knife CLI for agile DevOps teams. Keep backlog, specs, tests, and code in sync with validation and contract enforcement for new projects and long-lived codebases." readme = "README.md" requires-python = ">=3.11" diff --git a/setup.py b/setup.py index 8d01d675..79e2e3cd 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ if __name__ == "__main__": _setup = setup( name="specfact-cli", - version="0.31.0", + version="0.31.1", description=( "The swiss knife CLI for agile DevOps teams. Keep backlog, specs, tests, and code in sync with " "validation and contract enforcement for new projects and long-lived codebases." diff --git a/src/__init__.py b/src/__init__.py index 747c8b7d..5b0b4fa0 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -3,4 +3,4 @@ """ # Package version: keep in sync with pyproject.toml, setup.py, src/specfact_cli/__init__.py -__version__ = "0.31.0" +__version__ = "0.31.1" diff --git a/src/specfact_cli/__init__.py b/src/specfact_cli/__init__.py index 4944df2d..e50a89f9 100644 --- a/src/specfact_cli/__init__.py +++ b/src/specfact_cli/__init__.py @@ -8,6 +8,6 @@ - Supporting agile ceremonies and team workflows """ -__version__ = "0.31.0" +__version__ = "0.31.1" __all__ = ["__version__"] diff --git a/tools/smart_test_coverage.py b/tools/smart_test_coverage.py index e4a7e651..dfde8d89 100755 --- a/tools/smart_test_coverage.py +++ b/tools/smart_test_coverage.py @@ -144,22 +144,23 @@ def __init__(self, project_root: str = ".", coverage_threshold: float | None = N def _build_hatch_test_cmd( self, with_coverage: bool, extra_args: list[str] | None = None, parallel: bool = False ) -> list[str]: - """Construct the hatch test command, honoring optional env selection.""" - base_cmd: list[str] = [self.hatch_bin, "test"] - if self.hatch_test_env: - env_name = ( - self.hatch_test_env - if self.hatch_test_env.startswith("hatch-test.") - else f"hatch-test.{self.hatch_test_env}" - ) - base_cmd += ["-e", env_name] - if with_coverage: - base_cmd += ["--cover"] - # Pass pytest args explicitly after `--` to avoid collisions with hatch-test flags - # (e.g., hatch's `-r/--randomize` conflicts with pytest `-r` report option). + """Construct the hatch run command for the test script, honoring optional env selection. + + Uses `hatch run -e ENV run-cov` (or `run`) so that pytest receives only script + args after `--`. Using `hatch test -e ENV --cover` can forward `-e` to pytest in + some environments and cause "unrecognized arguments: -e". + """ + if self.hatch_test_env and self.hatch_test_env.startswith("hatch-test."): + env_name = self.hatch_test_env + elif self.hatch_test_env: + env_name = f"hatch-test.{self.hatch_test_env}" + else: + env_name = "hatch-test.py3.12" + script = "run-cov" if with_coverage else "run" + # -e/--env must be the first option before the run command + base_cmd: list[str] = [self.hatch_bin, "-e", env_name, "run", script] + # Pass pytest args explicitly after `--` to avoid collisions with hatch flags. base_cmd += ["--", "-v", "-r", "fEw"] - # Parallel execution is handled by hatch configuration (parallel = true) - # No need to add -n parameter manually if extra_args: base_cmd += extra_args return base_cmd