From 541d801dbb1afb6b007d6907e639ff10a3e6c308 Mon Sep 17 00:00:00 2001 From: Mara Nikola Kiefer Date: Tue, 27 Jan 2026 23:02:05 +0100 Subject: [PATCH] chore: enhance campaign id handling --- .../security-alert-burndown.lock.yml | 717 +++++++++++++++++- .github/workflows/security-alert-burndown.md | 3 +- pkg/campaign/injection.go | 70 +- pkg/campaign/injection_test.go | 60 ++ .../compiler_orchestrator_workflow.go | 2 + pkg/workflow/compiler_types.go | 1 + pkg/workflow/frontmatter_types.go | 23 +- pkg/workflow/frontmatter_types_test.go | 19 + 8 files changed, 885 insertions(+), 10 deletions(-) create mode 100644 pkg/campaign/injection_test.go diff --git a/.github/workflows/security-alert-burndown.lock.yml b/.github/workflows/security-alert-burndown.lock.yml index d583d52f31..a62c6fc842 100644 --- a/.github/workflows/security-alert-burndown.lock.yml +++ b/.github/workflows/security-alert-burndown.lock.yml @@ -147,7 +147,7 @@ jobs: const determineAutomaticLockdown = require('/opt/gh-aw/actions/determine_automatic_lockdown.cjs'); await determineAutomaticLockdown(github, context, core); - name: Download container images - run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/github-mcp-server:v0.30.1 ghcr.io/githubnext/gh-aw-mcpg:v0.0.82 node:lts-alpine + run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/github-mcp-server:v0.30.2 ghcr.io/githubnext/gh-aw-mcpg:v0.0.84 node:lts-alpine - name: Write Safe Outputs Config run: | mkdir -p /opt/gh-aw/safeoutputs @@ -470,7 +470,7 @@ jobs: # Register API key as secret to mask it from logs echo "::add-mask::${MCP_GATEWAY_API_KEY}" export GH_AW_ENGINE="copilot" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e DEBUG="*" -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/githubnext/gh-aw-mcpg:v0.0.82' + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e DEBUG="*" -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/githubnext/gh-aw-mcpg:v0.0.84' mkdir -p /home/runner/.copilot cat << MCPCONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh @@ -478,7 +478,7 @@ jobs: "mcpServers": { "github": { "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v0.30.1", + "container": "ghcr.io/github/github-mcp-server:v0.30.2", "env": { "GITHUB_LOCKDOWN_MODE": "$GITHUB_MCP_LOCKDOWN", "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", @@ -530,7 +530,7 @@ jobs: allowed_domains: ["defaults"], firewall_enabled: true, awf_version: "v0.11.2", - awmg_version: "v0.0.82", + awmg_version: "v0.0.84", steps: { firewall: "squid" }, @@ -641,6 +641,715 @@ jobs: - Copilot agent reviews and processes assigned PRs - Project board reflects current state of dependency updates + + + --- + # WORKFLOW EXECUTION (PHASE 0) + --- + # Workflow Execution + + This campaign references the following campaign workers. These workers follow the first-class worker pattern: they are dispatch-only workflows with standardized input contracts. + + **IMPORTANT: Workers are orchestrated, not autonomous. They accept `campaign_id` and `payload` inputs via workflow_dispatch.** + + --- + + ## Campaign Workers + + + + **Worker Pattern**: All workers MUST: + - Use `workflow_dispatch` as the ONLY trigger (no schedule/push/pull_request) + - Accept `campaign_id` (string) and `payload` (string; JSON) inputs + - Implement idempotency via deterministic work item keys + - Label all created items with `z_campaign_security-alert-burndown` + + --- + + ## Workflow Creation Guardrails + + ### Before Creating Any Worker Workflow, Ask: + + 1. **Does this workflow already exist?** - Check `.github/workflows/` thoroughly + 2. **Can an existing workflow be adapted?** - Even if not perfect, existing is safer + 3. **Is the requirement clear?** - Can you articulate exactly what it should do? + 4. **Is it testable?** - Can you verify it works with test inputs? + 5. **Is it reusable?** - Could other campaigns benefit from this worker? + + ### Only Create New Workers When: + + ✅ **All these conditions are met:** + - No existing workflow does the required task + - The campaign objective explicitly requires this capability + - You have a clear, specific design for the worker + - The worker has a focused, single-purpose scope + - You can test it independently before campaign use + + ❌ **Never create workers when:** + - You're unsure about requirements + - An existing workflow "mostly" works + - The worker would be complex or multi-purpose + - You haven't verified it doesn't already exist + - You can't clearly explain what it does in one sentence + + --- + + ## Worker Creation Template + + If you must create a new worker (only after checking ALL guardrails above), use this template: + + **Create the workflow file at `.github/workflows/.md`:** + + ```yaml + --- + name: + description: + + on: + workflow_dispatch: + inputs: + campaign_id: + description: 'Campaign identifier' + required: true + type: string + payload: + description: 'JSON payload with work item details' + required: true + type: string + + tracker-id: + + tools: + github: + toolsets: [default] + # Add minimal additional tools as needed + + safe-outputs: + create-pull-request: + max: 1 # Start conservative + add-comment: + max: 2 + --- + + # + + You are a campaign worker that processes work items. + + ## Input Contract + + Parse inputs: + ```javascript + const campaignId = context.payload.inputs.campaign_id; + const payload = JSON.parse(context.payload.inputs.payload); + ``` + + Expected payload structure: + ```json + { + "repository": "owner/repo", + "work_item_id": "unique-id", + "target_ref": "main", + // Additional context... + } + ``` + + ## Idempotency Requirements + + 1. **Generate deterministic key**: + ``` + const workKey = `campaign-${campaignId}-${payload.repository}-${payload.work_item_id}`; + ``` + + 2. **Check for existing work**: + - Search for PRs/issues with `workKey` in title + - Filter by label: `z_campaign_${campaignId}` + - If found: Skip or update + - If not: Create new + + 3. **Label all created items**: + - Apply `z_campaign_${campaignId}` label + - This enables discovery by orchestrator + + ## Task + + + + ## Output + + Report: + - Link to created/updated PR or issue + - Whether work was skipped (exists) or completed + - Any errors or blockers + ``` + + **After creating:** + - Compile: `gh aw compile .md` + - **CRITICAL: Test with sample inputs** (see testing requirements below) + + --- + + ## Worker Testing (MANDATORY) + + **Why test?** - Untested workers may fail during campaign execution. Test with sample inputs first to catch issues early. + + **Testing steps:** + + 1. **Prepare test payload**: + ```json + { + "repository": "test-org/test-repo", + "work_item_id": "test-1", + "target_ref": "main" + } + ``` + + 2. **Trigger test run**: + ```bash + gh workflow run .yml \ + -f campaign_id=security-alert-burndown \ + -f payload='{"repository":"test-org/test-repo","work_item_id":"test-1"}' + ``` + + Or via GitHub MCP: + ```javascript + mcp__github__run_workflow( + workflow_id: "", + ref: "main", + inputs: { + campaign_id: "security-alert-burndown", + payload: JSON.stringify({repository: "test-org/test-repo", work_item_id: "test-1"}) + } + ) + ``` + + 3. **Wait for completion**: Poll until status is "completed" + + 4. **Verify success**: + - Check that workflow succeeded + - Verify idempotency: Run again with same inputs, should skip/update + - Review created items have correct labels + - Confirm deterministic keys are used + + 5. **Test failure actions**: + - DO NOT use the worker if testing fails + - Analyze failure logs + - Make corrections + - Recompile and retest + - If unfixable after 2 attempts, report in status and skip + + **Note**: Workflows that accept `workflow_dispatch` inputs can receive parameters from the orchestrator. This enables the orchestrator to provide context, priorities, or targets based on its decisions. See [DispatchOps documentation](https://githubnext.github.io/gh-aw/guides/dispatchops/#with-input-parameters) for input parameter examples. + + --- + + ## Orchestration Guidelines + + **Execution pattern:** + - Workers are **orchestrated, not autonomous** + - Orchestrator discovers work items via discovery manifest + - Orchestrator decides which workers to run and with what inputs + - Workers receive `campaign_id` and `payload` via workflow_dispatch + - Sequential vs parallel execution is orchestrator's decision + + **Worker dispatch:** + - Parse discovery manifest (`./.gh-aw/campaign.discovery.json`) + - For each work item needing processing: + 1. Determine appropriate worker for this item type + 2. Construct payload with work item details + 3. Dispatch worker via workflow_dispatch with campaign_id and payload + 4. Track dispatch status + + **Input construction:** + ```javascript + // Example: Dispatching security-fix worker + const workItem = discoveryManifest.items[0]; + const payload = { + repository: workItem.repo, + work_item_id: `alert-${workItem.number}`, + target_ref: "main", + alert_type: "sql-injection", + file_path: "src/db.go", + line_number: 42 + }; + + await github.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: "security-fix-worker.yml", + ref: "main", + inputs: { + campaign_id: "security-alert-burndown", + payload: JSON.stringify(payload) + } + }); + ``` + + **Idempotency by design:** + - Workers implement their own idempotency checks + - Orchestrator doesn't need to track what's been processed + - Can safely re-dispatch work items across runs + - Workers will skip or update existing items + + **Failure handling:** + - If a worker dispatch fails, note it but continue + - Worker failures don't block entire campaign + - Report all failures in status update with context + - Humans can intervene if needed + + --- + + ## After Worker Orchestration + + Once workers have been dispatched (or new workers created and tested), proceed with normal orchestrator steps: + + 1. **Discovery** - Read state from discovery manifest and project board + 2. **Planning** - Determine what needs updating on project board + 3. **Project Updates** - Write state changes to project board + 4. **Status Reporting** - Report progress, worker dispatches, failures, next steps + + --- + + ## Key Differences from Fusion Approach + + **Old fusion approach (REMOVED)**: + - Workers had mixed triggers (schedule + workflow_dispatch) + - Fusion dynamically added workflow_dispatch to existing workflows + - Workers stored in campaign-specific folders + - Ambiguous ownership and trigger precedence + + **New first-class worker approach**: + - Workers are dispatch-only (on: workflow_dispatch) + - Standardized input contract (campaign_id, payload) + - Explicit idempotency via deterministic keys + - Clear ownership: workers are orchestrated, not autonomous + - Workers stored with regular workflows (not campaign-specific folders) + - Orchestration policy kept explicit in orchestrator + + This eliminates duplicate execution problems and makes orchestration concerns explicit. + --- + # ORCHESTRATOR INSTRUCTIONS + --- + # Orchestrator Instructions + + This orchestrator coordinates a single campaign by discovering worker outputs and making deterministic decisions. + + **Scope:** orchestration only (discovery, planning, pacing, reporting). + **Actuation model:** **dispatch-only** — the orchestrator may only act by dispatching allowlisted worker workflows. + **Write authority:** all GitHub writes (Projects, issues/PRs, comments, status updates) must happen in worker workflows. + + --- + + ## Traffic and Rate Limits (Required) + + - Minimize API calls; avoid full rescans when possible. + - Prefer incremental discovery with deterministic ordering (e.g., by `updatedAt`, tie-break by ID). + - Enforce strict pagination budgets; if a query requires many pages, stop early and continue next run. + - Use a durable cursor/checkpoint so the next run continues without rescanning. + - On throttling (HTTP 429 / rate-limit 403), do not retry aggressively; back off and end the run after reporting what remains. + + + **Cursor file (repo-memory)**: `memory/campaigns/security-alert-burndown/cursor.json` + **File system path**: `/tmp/gh-aw/repo-memory/campaigns/security-alert-burndown/cursor.json` + - If it exists: read first and continue from its boundary. + - If it does not exist: create it by end of run. + - Always write the updated cursor back to the same path. + + + + **Metrics snapshots (repo-memory)**: `memory/campaigns/security-alert-burndown/metrics/*.json` + **File system path**: `/tmp/gh-aw/repo-memory/campaigns/security-alert-burndown/metrics/*.json` + - Persist one append-only JSON metrics snapshot per run (new file per run; do not rewrite history). + - Use UTC date (`YYYY-MM-DD`) in the filename (example: `metrics/2025-12-22.json`). + + + + + + --- + + ## Core Principles + + 1. Workers are immutable and campaign-agnostic + 2. The GitHub Project board is the authoritative campaign state + 3. Correlation is explicit (tracker-id AND labels) + 4. Reads and writes are separate steps (never interleave) + 5. Idempotent operation is mandatory (safe to re-run) + 6. Orchestrators do not write GitHub state directly + + --- + + ## Execution Steps (Required Order) + + ### Step 1 — Read State (Discovery) [NO WRITES] + + **IMPORTANT**: Discovery has been precomputed. Read the discovery manifest instead of performing GitHub-wide searches. + + 1) Read the precomputed discovery manifest: `./.gh-aw/campaign.discovery.json` + + 2) Parse discovered items from the manifest: + - Each item has: url, content_type (issue/pull_request/discussion), number, repo, created_at, updated_at, state + - Closed items have: closed_at (for issues) or merged_at (for PRs) + - Items are pre-sorted by updated_at for deterministic processing + + 3) Check the manifest summary for work counts. + + 4) Discovery cursor is maintained automatically in repo-memory; do not modify it manually. + + ### Step 2 — Make Decisions (Planning) [NO WRITES] + + 5) Determine desired `status` strictly from explicit GitHub state: + - Open → `Todo` (or `In Progress` only if explicitly indicated elsewhere) + - Closed (issue/discussion) → `Done` + - Merged (PR) → `Done` + + 6) Calculate required date fields (for workers that sync Projects): + - `start_date`: format `created_at` as `YYYY-MM-DD` + - `end_date`: + - if closed/merged → format `closed_at`/`merged_at` as `YYYY-MM-DD` + - if open → **today's date** formatted `YYYY-MM-DD` + + 7) Reads and writes are separate steps (never interleave). + + ### Step 3 — Dispatch Workers (Execution) [DISPATCH ONLY] + + 8) For each selected unit of work, dispatch a worker workflow using `dispatch-workflow`. + + Constraints: + - Only dispatch allowlisted workflows. + - Keep within the dispatch-workflow max for this run. + + ### Step 4 — Report (No Writes) + + 9) Summarize what you dispatched, what remains, and what should run next. + + If a status update is required on the GitHub Project, dispatch a dedicated reporting/sync worker to perform that write. + + **Discovered:** 25 items (15 issues, 10 PRs) + **Processed:** 10 items added to project, 5 updated + **Completion:** 60% (30/50 total tasks) + + ## Most Important Findings + + 1. **Critical accessibility gaps identified**: 3 high-severity accessibility issues discovered in mobile navigation, requiring immediate attention + 2. **Documentation coverage acceleration**: Achieved 5% improvement in one week (best velocity so far) + 3. **Worker efficiency improving**: daily-doc-updater now processing 40% more items per run + + ## What Was Learned + + - Multi-device testing reveals issues that desktop-only testing misses - should be prioritized + - Documentation updates tied to code changes have higher accuracy and completeness + - Users report fewer issues when examples include error handling patterns + + ## Campaign Progress + + **Documentation Coverage** (Primary Metric): + - Baseline: 85% → Current: 88% → Target: 95% + - Direction: ↑ Increasing (+3% this week, +1% velocity/week) + - Status: ON TRACK - At current velocity, will reach 95% in 7 weeks + + **Accessibility Score** (Supporting Metric): + - Baseline: 90% → Current: 91% → Target: 98% + - Direction: ↑ Increasing (+1% this month) + - Status: AT RISK - Slower progress than expected, may need dedicated focus + + **User-Reported Issues** (Supporting Metric): + - Baseline: 15/month → Current: 12/month → Target: 5/month + - Direction: ↓ Decreasing (-3 this month, -20% velocity) + - Status: ON TRACK - Trending toward target + + ## Next Steps + + 1. Address 3 critical accessibility issues identified this run (high priority) + 2. Continue processing remaining 15 discovered items + 3. Focus on accessibility improvements to accelerate supporting KPI + 4. Maintain current documentation coverage velocity + ``` + + 12) Report: + - counts discovered (by type) + - counts processed this run (by action: add/status_update/backfill/noop/failed) + - counts deferred due to budgets + - failures (with reasons) + - completion state (work items only) + - cursor advanced / remaining backlog estimate + + --- + + ## Authority + + If any instruction in this file conflicts with **Project Update Instructions**, the Project Update Instructions win for all project writes. + --- + # PROJECT UPDATE INSTRUCTIONS (AUTHORITATIVE FOR WRITES) + --- + # Project Update Instructions (Authoritative Write Contract) + + ## Project Board Integration + + This file defines the ONLY allowed rules for writing to the GitHub Project board. + If any other instructions conflict with this file, THIS FILE TAKES PRECEDENCE for all project writes. + + --- + + ## 0) Hard Requirements (Do Not Deviate) + + - Orchestrators are dispatch-only and MUST NOT perform project writes directly. + - Worker workflows performing project writes MUST use only the `update-project` safe-output. + - All writes MUST target exactly: + - **Project URL**: `https://github.com/orgs/githubnext/projects/144` + - Every item MUST include: + - `campaign_id: "security-alert-burndown"` + + ## Campaign ID + + PROMPT_EOF + cat << 'PROMPT_EOF' >> "$GH_AW_PROMPT" + All campaign tracking MUST key off `campaign_id: "security-alert-burndown"`. + + --- + + ## 1) Required Project Fields (Must Already Exist) + + | Field | Type | Allowed / Notes | + |---|---|---| + | `status` | single-select | `Todo` / `In Progress` / `Review required` / `Blocked` / `Done` | + | `campaign_id` | text | Must equal `security-alert-burndown` | + | `worker_workflow` | text | workflow ID or `"unknown"` | + | `target_repo` | text | `owner/repo` | + | `priority` | single-select | `High` / `Medium` / `Low` | + | `size` | single-select | `Small` / `Medium` / `Large` | + | `start_date` | date | `YYYY-MM-DD` | + | `end_date` | date | `YYYY-MM-DD` | + + Field names are case-sensitive. + + --- + + ## 2) Content Identification (Mandatory) + + Use **content number** (integer), never the URL as an identifier. + + - Issue URL: `.../issues/123` → `content_type: "issue"`, `content_number: 123` + - PR URL: `.../pull/456` → `content_type: "pull_request"`, `content_number: 456` + + --- + + ## 3) Deterministic Field Rules (No Inference) + + These rules apply to any time you write fields: + + - `campaign_id`: always `security-alert-burndown` + - `worker_workflow`: workflow ID if known, else `"unknown"` + - `target_repo`: extract `owner/repo` from the issue/PR URL + - `priority`: default `Medium` unless explicitly known + - `size`: default `Medium` unless explicitly known + - `start_date`: issue/PR `created_at` formatted `YYYY-MM-DD` + - `end_date`: + - if closed/merged → `closed_at` / `merged_at` formatted `YYYY-MM-DD` + - if open → **today’s date** formatted `YYYY-MM-DD` (**required for roadmap view; do not leave blank**) + + For open items, `end_date` is a UI-required placeholder and does NOT represent actual completion. + + --- + + ## 4) Read-Write Separation (Prevents Read/Write Mixing) + + 1. **READ STEP (no writes)** — validate existence and gather metadata + 2. **WRITE STEP (writes only)** — execute `update-project` + + Never interleave reads and writes. + + --- + + ## 5) Adding an Issue or PR (First Write) + + ### Adding New Issues + + When first adding an item to the project, you MUST write ALL required fields. + + ```yaml + update-project: + project: "https://github.com/orgs/githubnext/projects/144" + campaign_id: "security-alert-burndown" + content_type: "issue" # or "pull_request" + content_number: 123 + fields: + status: "Todo" # "Done" if already closed/merged + campaign_id: "security-alert-burndown" + worker_workflow: "unknown" + target_repo: "owner/repo" + priority: "Medium" + size: "Medium" + start_date: "2025-12-15" + end_date: "2026-01-03" + ``` + + --- + + ## 6) Updating an Existing Item (Minimal Writes) + + ### Updating Existing Items + + Preferred behavior is minimal, idempotent writes: + + - If item exists and `status` is unchanged → **No-op** + - If item exists and `status` differs → **Update `status` only** + - If any required field is missing/empty/invalid → **One-time full backfill** (repair only) + + ### Status-only Update (Default) + + ```yaml + update-project: + project: "https://github.com/orgs/githubnext/projects/144" + campaign_id: "security-alert-burndown" + content_type: "issue" # or "pull_request" + content_number: 123 + fields: + status: "Done" + ``` + + ### Full Backfill (Repair Only) + + ```yaml + update-project: + project: "https://github.com/orgs/githubnext/projects/144" + campaign_id: "security-alert-burndown" + content_type: "issue" # or "pull_request" + content_number: 123 + fields: + status: "Done" + campaign_id: "security-alert-burndown" + worker_workflow: "WORKFLOW_ID" + target_repo: "owner/repo" + priority: "Medium" + size: "Medium" + start_date: "2025-12-15" + end_date: "2026-01-02" + ``` + + --- + + ## 7) Idempotency Rules + + - Matching status already set → **No-op** + - Different status → **Status-only update** + - Invalid/deleted/inaccessible URL → **Record failure and continue** + + ## Write Operation Rules + + All writes MUST conform to this file and use `update-project` only. + + --- + + ## 8) Logging + Failure Handling (Mandatory) + + For every attempted item, record: + + - `content_type`, `content_number`, `target_repo` + - action taken: `noop | add | status_update | backfill | failed` + - error details if failed + + Failures must not stop processing remaining items. + + --- + + ## 9) Worker Workflow Policy + + - Workers are campaign-agnostic. + - Orchestrator populates `worker_workflow`. + - If `worker_workflow` cannot be determined, it MUST remain `"unknown"` unless explicitly reclassified by the orchestrator. + + --- + + ## 10) Parent / Sub-Issue Rules (Campaign Hierarchy) + + - Each project board MUST have exactly **one Epic issue** representing the campaign. + - The Epic issue MUST: + - Be added to the project board + - Use the same `campaign_id` + - Use `worker_workflow: "unknown"` + + - All campaign work issues (non-epic) MUST be created as **sub-issues of the Epic**. + - Issues MUST NOT be re-parented based on worker assignment. + + - Pull requests cannot be sub-issues: + - PRs MUST reference their related issue via standard GitHub linking (e.g. “Closes #123”). + + - Worker grouping MUST be done via the `worker_workflow` project field, not via parent issues. + + - The Epic issue is narrative only. + - The project board is the sole authoritative source of campaign state. + + --- + + ## Appendix — Machine Check Checklist (Optional) + + This checklist is designed to validate outputs before executing project writes. + + ### A) Output Structure Checks + + - [ ] All writes use `update-project:` blocks (no other write mechanism). + - [ ] Each `update-project` block includes: + - [ ] `project: "https://github.com/orgs/githubnext/projects/144"` + - [ ] `campaign_id: "security-alert-burndown"` (top-level) + - [ ] `content_type` ∈ {`issue`, `pull_request`} + - [ ] `content_number` is an integer + - [ ] `fields` object is present + + ### B) Field Validity Checks + + - [ ] `fields.status` ∈ {`Todo`, `In Progress`, `Review required`, `Blocked`, `Done`} + - [ ] `fields.campaign_id` is present on first-add/backfill and equals `security-alert-burndown` + - [ ] `fields.worker_workflow` is present on first-add/backfill and is either a known workflow ID or `"unknown"` + - [ ] `fields.target_repo` matches `owner/repo` + - [ ] `fields.priority` ∈ {`High`, `Medium`, `Low`} + - [ ] `fields.size` ∈ {`Small`, `Medium`, `Large`} + - [ ] `fields.start_date` matches `YYYY-MM-DD` + - [ ] `fields.end_date` matches `YYYY-MM-DD` + + ### C) Update Semantics Checks + + - [ ] For existing items, payload is **status-only** unless explicitly doing a backfill repair. + - [ ] Backfill is used only when required fields are missing/empty/invalid. + - [ ] No payload overwrites `priority`/`size`/`worker_workflow` with defaults during a normal status update. + + ### D) Read-Write Separation Checks + + - [ ] All reads occur before any writes (no read/write interleaving). + - [ ] Writes are batched separately from discovery. + + ### E) Epic/Hierarchy Checks (Policy-Level) + + - [ ] Exactly one Epic exists for the campaign board. + - [ ] Epic is on the board and uses `worker_workflow: "unknown"`. + - [ ] All campaign work issues are sub-issues of the Epic (if supported by environment/tooling). + - [ ] PRs are linked to issues via GitHub linking (e.g. “Closes #123”). + + ### F) Failure Handling Checks + + - [ ] Invalid/deleted/inaccessible items are logged as failures and processing continues. + - [ ] Idempotency is delegated to the `update-project` tool; no pre-filtering by board presence. + --- + # CLOSING INSTRUCTIONS (HIGHEST PRIORITY) + --- + # Closing Instructions (Highest Priority) + + Execute all four steps in strict order: + + 1. Read State (no writes) + 2. Make Decisions (no writes) + 3. Dispatch Workers (dispatch-workflow only) + 4. Report + + The following rules are mandatory and override inferred behavior: + + - The GitHub Project board is the single source of truth. + - All project writes MUST comply with the Project Update Instructions (in workers). + - State reads and state writes MUST NOT be interleaved. + - Do NOT infer missing data or invent values. + - Do NOT reorganize hierarchy. + - Do NOT overwrite fields except as explicitly allowed. + - Workers are immutable and campaign-agnostic. + + If any instruction conflicts, the Project Update Instructions take precedence for all writes. PROMPT_EOF - name: Substitute placeholders uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 diff --git a/.github/workflows/security-alert-burndown.md b/.github/workflows/security-alert-burndown.md index f2f5f4e3b9..987d53b542 100644 --- a/.github/workflows/security-alert-burndown.md +++ b/.github/workflows/security-alert-burndown.md @@ -9,7 +9,8 @@ permissions: issues: read pull-requests: read contents: read -project: https://github.com/orgs/githubnext/projects/144 +project: + url: https://github.com/orgs/githubnext/projects/144 --- # Security Alert Burndown Campaign diff --git a/pkg/campaign/injection.go b/pkg/campaign/injection.go index 53235d0b4b..a53f2836ec 100644 --- a/pkg/campaign/injection.go +++ b/pkg/campaign/injection.go @@ -2,6 +2,7 @@ package campaign import ( "fmt" + "regexp" "strings" "github.com/githubnext/gh-aw/pkg/logger" @@ -10,6 +11,18 @@ import ( var injectionLog = logger.New("campaign:injection") +var campaignIDSanitizer = regexp.MustCompile(`[^a-z0-9-]+`) + +func normalizeCampaignID(id string) string { + // Keep IDs stable and safe for labels/paths. + id = strings.ToLower(strings.TrimSpace(id)) + id = strings.ReplaceAll(id, "_", "-") + id = strings.ReplaceAll(id, " ", "-") + id = campaignIDSanitizer.ReplaceAllString(id, "-") + id = strings.Trim(id, "-") + return id +} + // InjectOrchestratorFeatures detects if a workflow has project field with campaign // configuration and injects orchestrator features directly into the workflow during compilation. // This transforms the workflow into a campaign orchestrator without generating separate files. @@ -24,20 +37,42 @@ func InjectOrchestratorFeatures(workflowData *workflow.WorkflowData) error { project := workflowData.ParsedFrontmatter.Project + // Determine whether the project config looks like "project tracking" only. + // A minimal campaign can specify only the project URL (either short or long form) and omit + // campaign fields like id/workflows; in that case we infer the campaign ID from the workflow filename. + hasTrackingOnlySettings := len(project.Scope) > 0 || + project.MaxUpdates > 0 || + project.MaxStatusUpdates > 0 || + strings.TrimSpace(project.GitHubToken) != "" || + project.DoNotDowngradeDoneItems != nil + // Check if project has any campaign orchestration fields to determine if this is a campaign // Campaign indicators (any of these present means it's a campaign orchestrator): + // - explicit campaign ID // - workflows list (predefined workers) // - governance policies (campaign-specific constraints) // - bootstrap configuration (initial work item generation) // - memory-paths, metrics-glob, cursor-glob (campaign state tracking) // If only URL and scope are present, it's simple project tracking, not a campaign - isCampaign := len(project.Workflows) > 0 || + hasCampaignIndicators := strings.TrimSpace(project.ID) != "" || + len(project.Workflows) > 0 || project.Governance != nil || project.Bootstrap != nil || len(project.MemoryPaths) > 0 || project.MetricsGlob != "" || project.CursorGlob != "" + // If the user used the object form with only a URL (no tracking-only knobs), treat it as a campaign + // and infer the campaign ID from the workflow filename (minus .md). + if !hasCampaignIndicators && !hasTrackingOnlySettings { + if workflowData.WorkflowID != "" { + project.ID = workflowData.WorkflowID + hasCampaignIndicators = true + } + } + + isCampaign := hasCampaignIndicators + if !isCampaign { injectionLog.Print("Project field present but no campaign indicators, treating as simple project tracking") return nil @@ -46,10 +81,37 @@ func InjectOrchestratorFeatures(workflowData *workflow.WorkflowData) error { injectionLog.Printf("Detected campaign orchestrator: workflows=%d, has_governance=%v, has_bootstrap=%v", len(project.Workflows), project.Governance != nil, project.Bootstrap != nil) - // Derive campaign ID from workflow name or use explicit ID - campaignID := workflowData.FrontmatterName - if project.ID != "" { + // Derive campaign ID (prefer explicit id, then workflow filename, then workflow name). + // Note: workflowData.FrontmatterName is the *frontmatter name field* (display name), not the file basename. + campaignID := "" + if strings.TrimSpace(project.ID) != "" { campaignID = project.ID + } else if strings.TrimSpace(workflowData.WorkflowID) != "" { + campaignID = workflowData.WorkflowID + } else if strings.TrimSpace(workflowData.Name) != "" { + campaignID = workflowData.Name + } else { + campaignID = "campaign" + } + campaignID = normalizeCampaignID(campaignID) + if campaignID == "" { + campaignID = "campaign" + } + + // Apply campaign defaults (matching the historical .campaign.md defaults) when omitted. + // This keeps project-based campaigns minimal: users can specify just url + id. + project.ID = campaignID + if strings.TrimSpace(project.TrackerLabel) == "" { + project.TrackerLabel = fmt.Sprintf("z_campaign_%s", campaignID) + } + if len(project.MemoryPaths) == 0 { + project.MemoryPaths = []string{fmt.Sprintf("memory/campaigns/%s/**", campaignID)} + } + if strings.TrimSpace(project.MetricsGlob) == "" { + project.MetricsGlob = fmt.Sprintf("memory/campaigns/%s/metrics/*.json", campaignID) + } + if strings.TrimSpace(project.CursorGlob) == "" { + project.CursorGlob = fmt.Sprintf("memory/campaigns/%s/cursor.json", campaignID) } // Build campaign prompt data from project configuration diff --git a/pkg/campaign/injection_test.go b/pkg/campaign/injection_test.go new file mode 100644 index 0000000000..ff135d6d79 --- /dev/null +++ b/pkg/campaign/injection_test.go @@ -0,0 +1,60 @@ +package campaign + +import ( + "testing" + + "github.com/githubnext/gh-aw/pkg/workflow" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInjectOrchestratorFeatures_ProjectIDTriggersCampaignAndDefaults(t *testing.T) { + data := &workflow.WorkflowData{ + Name: "Security Alert Burndown", + WorkflowID: "security-alert-burndown", + MarkdownContent: "# Test", + ParsedFrontmatter: &workflow.FrontmatterConfig{ + Project: &workflow.ProjectConfig{ + URL: "https://github.com/orgs/githubnext/projects/144", + }, + }, + } + + err := InjectOrchestratorFeatures(data) + require.NoError(t, err, "campaign injection should succeed") + require.NotNil(t, data.ParsedFrontmatter, "ParsedFrontmatter should remain") + require.NotNil(t, data.ParsedFrontmatter.Project, "Project should remain") + + project := data.ParsedFrontmatter.Project + assert.Equal(t, "security-alert-burndown", project.ID, "Campaign ID should be inferred and normalized") + assert.Equal(t, "z_campaign_security-alert-burndown", project.TrackerLabel, "TrackerLabel should be defaulted") + assert.Equal(t, []string{"memory/campaigns/security-alert-burndown/**"}, project.MemoryPaths, "MemoryPaths should be defaulted") + assert.Equal(t, "memory/campaigns/security-alert-burndown/metrics/*.json", project.MetricsGlob, "MetricsGlob should be defaulted") + assert.Equal(t, "memory/campaigns/security-alert-burndown/cursor.json", project.CursorGlob, "CursorGlob should be defaulted") + + assert.Contains(t, data.MarkdownContent, "ORCHESTRATOR INSTRUCTIONS", "Markdown should have orchestrator instructions injected") +} + +func TestInjectOrchestratorFeatures_ProjectTrackingOnly_DoesNotInject(t *testing.T) { + data := &workflow.WorkflowData{ + Name: "Project Tracking Only", + FrontmatterName: "project-tracking-only", + MarkdownContent: "# Test", + ParsedFrontmatter: &workflow.FrontmatterConfig{ + Project: &workflow.ProjectConfig{ + URL: "https://github.com/orgs/githubnext/projects/144", + Scope: []string{"githubnext/gh-aw"}, + }, + }, + } + + err := InjectOrchestratorFeatures(data) + require.NoError(t, err, "non-campaign project tracking should be a no-op") + assert.NotContains(t, data.MarkdownContent, "ORCHESTRATOR INSTRUCTIONS", "Should not inject campaign sections") +} + +func TestNormalizeCampaignID(t *testing.T) { + assert.Equal(t, "security-alert-burndown", normalizeCampaignID("Security Alert Burndown")) + assert.Equal(t, "security-alert-burndown", normalizeCampaignID("security_alert_burndown")) + assert.Equal(t, "security-alert-burndown", normalizeCampaignID(" security---alert@@burndown ")) +} diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go index 2b699c75ca..f4f44ec218 100644 --- a/pkg/workflow/compiler_orchestrator_workflow.go +++ b/pkg/workflow/compiler_orchestrator_workflow.go @@ -49,6 +49,8 @@ func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) // Build initial workflow data structure workflowData := c.buildInitialWorkflowData(result, toolsResult, engineSetup, engineSetup.importsResult) + // Store a stable workflow identifier derived from the file name. + workflowData.WorkflowID = GetWorkflowIDFromPath(cleanPath) // Use shared action cache and resolver from the compiler actionCache, actionResolver := c.getSharedActionResolver() diff --git a/pkg/workflow/compiler_types.go b/pkg/workflow/compiler_types.go index 1887dffa0e..94f469e63d 100644 --- a/pkg/workflow/compiler_types.go +++ b/pkg/workflow/compiler_types.go @@ -279,6 +279,7 @@ type SkipIfNoMatchConfig struct { // WorkflowData holds all the data needed to generate a GitHub Actions workflow type WorkflowData struct { Name string + WorkflowID string // workflow identifier derived from markdown filename (basename without extension) TrialMode bool // whether the workflow is running in trial mode TrialLogicalRepo string // target repository slug for trial mode (owner/repo) FrontmatterName string // name field from frontmatter (for code scanning alert driver default) diff --git a/pkg/workflow/frontmatter_types.go b/pkg/workflow/frontmatter_types.go index d83d50a888..d9d24a4a40 100644 --- a/pkg/workflow/frontmatter_types.go +++ b/pkg/workflow/frontmatter_types.go @@ -3,6 +3,7 @@ package workflow import ( "encoding/json" "fmt" + "strings" "github.com/githubnext/gh-aw/pkg/logger" ) @@ -249,9 +250,29 @@ func ParseFrontmatterConfig(frontmatter map[string]any) (*FrontmatterConfig, err frontmatterTypesLog.Printf("Parsing frontmatter config with %d fields", len(frontmatter)) var config FrontmatterConfig + // Normalize mixed-type fields before unmarshaling into typed structs. + // In YAML frontmatter, "project" can be either: + // - a URL string (short form): project: https://github.com/orgs/.../projects/123 + // - an object (long form): project: { url: ... , ... } + // The typed struct expects an object, so convert the short form to the long form. + normalizedFrontmatter := make(map[string]any, len(frontmatter)) + for k, v := range frontmatter { + normalizedFrontmatter[k] = v + } + if projectValue, ok := frontmatter["project"]; ok { + if projectURL, ok := projectValue.(string); ok { + projectURL = strings.TrimSpace(projectURL) + if projectURL == "" { + delete(normalizedFrontmatter, "project") + } else { + normalizedFrontmatter["project"] = map[string]any{"url": projectURL} + } + } + } + // Use JSON marshaling for the entire frontmatter conversion // This automatically handles all field mappings - jsonBytes, err := json.Marshal(frontmatter) + jsonBytes, err := json.Marshal(normalizedFrontmatter) if err != nil { frontmatterTypesLog.Printf("Failed to marshal frontmatter: %v", err) return nil, fmt.Errorf("failed to marshal frontmatter to JSON: %w", err) diff --git a/pkg/workflow/frontmatter_types_test.go b/pkg/workflow/frontmatter_types_test.go index 9edd8cf382..e761bc1333 100644 --- a/pkg/workflow/frontmatter_types_test.go +++ b/pkg/workflow/frontmatter_types_test.go @@ -144,6 +144,25 @@ func TestParseFrontmatterConfig(t *testing.T) { } }) + t.Run("parses project as URL string (short form)", func(t *testing.T) { + frontmatter := map[string]any{ + "name": "project-short-form", + "project": "https://github.com/orgs/githubnext/projects/144", + } + + config, err := ParseFrontmatterConfig(frontmatter) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if config.Project == nil { + t.Fatal("Project should not be nil") + } + if config.Project.URL != "https://github.com/orgs/githubnext/projects/144" { + t.Errorf("Project.URL = %q, want %q", config.Project.URL, "https://github.com/orgs/githubnext/projects/144") + } + }) + t.Run("parses complete workflow config", func(t *testing.T) { frontmatter := map[string]any{ "name": "full-workflow",