From 570a57f7fc44621ee921e16b6c030cb50022199c Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Fri, 24 Apr 2026 16:53:56 +0200 Subject: [PATCH] chore(panel-review): tighten one-comment contract; safety cap 1 -> 7 Two coupled changes to the panel review workflow: 1. Output contract (Step 3): make it explicit that the single allowed comment MUST be the CEO's final synthesized verdict, not any sub-agent's intermediate output. The orchestrator is now instructed to tell each sub-agent: 'do not post any comment; return your findings to the orchestrator.' 2. Safe-outputs cap (frontmatter): bump add-comment max from 1 to 7. With the prompt-level contract above, the cap is just a fail-soft ceiling: when an LLM occasionally drifts and tries to emit an extra comment, the run no longer aborts at the safe-outputs gate and lose the verdict. Bloat removal: dropped the workflow-only ASCII rule from the agent prompt (it's a project source-code rule that doesn't apply to PR-comment prose) and the meta references to safe-outputs internals (the agent doesn't need to know it runs in gh-aw). Recompiled pr-review-panel.lock.yml via 'gh aw compile pr-review-panel'. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/pr-review-panel.lock.yml | 30 ++++++++--------- .github/workflows/pr-review-panel.md | 38 ++++++++++------------ 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/.github/workflows/pr-review-panel.lock.yml b/.github/workflows/pr-review-panel.lock.yml index d4ce34edf..2213f5c64 100644 --- a/.github/workflows/pr-review-panel.lock.yml +++ b/.github/workflows/pr-review-panel.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"568ddeef22451814cb5a691b0845f2c8ba599c72796186d9ad82b54ce1546571","compiler_version":"v0.68.3","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"7d07172bee3f1898f2e45a8bd63bdcb115a1ad4cee0285f6ec7df4b9b7c2da95","compiler_version":"v0.68.3","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_PLUGINS_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"ba90f2186d7ad780ec640f364005fa24e797b360","version":"v0.68.3"},{"repo":"microsoft/apm-action","sha":"9fe9337ef58b5e620e0113071ceb47a6a8a232f7","version":"v1.4.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.20"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.20"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.20"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.2.19"},{"image":"ghcr.io/github/github-mcp-server:v0.32.0"},{"image":"node:lts-alpine"}]} # ___ _ _ # / _ \ | | (_) @@ -197,16 +197,16 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_e279ddc7abe758c5_EOF' + cat << 'GH_AW_PROMPT_9b70ff9898f5f7d2_EOF' - GH_AW_PROMPT_e279ddc7abe758c5_EOF + GH_AW_PROMPT_9b70ff9898f5f7d2_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_e279ddc7abe758c5_EOF' + cat << 'GH_AW_PROMPT_9b70ff9898f5f7d2_EOF' - Tools: add_comment, missing_tool, missing_data, noop + Tools: add_comment(max:7), missing_tool, missing_data, noop The following GitHub context information is available for this workflow: @@ -236,15 +236,15 @@ jobs: {{/if}} - GH_AW_PROMPT_e279ddc7abe758c5_EOF + GH_AW_PROMPT_9b70ff9898f5f7d2_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_e279ddc7abe758c5_EOF' + cat << 'GH_AW_PROMPT_9b70ff9898f5f7d2_EOF' {{#runtime-import .github/workflows/pr-review-panel.md}} - GH_AW_PROMPT_e279ddc7abe758c5_EOF + GH_AW_PROMPT_9b70ff9898f5f7d2_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 @@ -437,15 +437,15 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_3e3c0744a2d8ea00_EOF' - {"add_comment":{"max":1},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_3e3c0744a2d8ea00_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_f819ea2bf38c25a8_EOF' + {"add_comment":{"max":7},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_f819ea2bf38c25a8_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | { "description_suffixes": { - "add_comment": " CONSTRAINTS: Maximum 1 comment(s) can be added. Supports reply_to_id for discussion threading." + "add_comment": " CONSTRAINTS: Maximum 7 comment(s) can be added. Supports reply_to_id for discussion threading." }, "repo_params": {}, "dynamic_tools": [] @@ -623,7 +623,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.19' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_9953eb7975809a34_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_2b3d00b802971123_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -664,7 +664,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_9953eb7975809a34_EOF + GH_AW_MCP_CONFIG_2b3d00b802971123_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -1289,7 +1289,7 @@ jobs: GH_AW_ALLOWED_DOMAINS: "*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":1},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":7},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/pr-review-panel.md b/.github/workflows/pr-review-panel.md index 878cd9b44..9f61833bd 100644 --- a/.github/workflows/pr-review-panel.md +++ b/.github/workflows/pr-review-panel.md @@ -17,7 +17,9 @@ description: Multi-persona expert panel review of labelled PRs, posting a single # `gh pr diff` which return inert text # - imports are pinned to microsoft/apm#main (panel skill + # persona definitions are trusted, not from the PR) -# - the only write surface is safe-outputs.add-comment (max 1) +# - the only write surface is safe-outputs.add-comment (max 7 +# is a safety ceiling; the agent is instructed to emit one +# synthesized verdict comment) # - `roles: [admin, maintainer, write]` ensures only repo # maintainers can trigger -- matches the trust model that # applying the `panel-review` label requires write access. @@ -73,7 +75,7 @@ network: safe-outputs: add-comment: - max: 1 + max: 7 timeout-minutes: 30 --- @@ -124,24 +126,20 @@ separate scope-analysis sub-agent for this. If the skill marks Auth Expert inactive, do not dispatch it; keep the Auth Expert heading in the final verdict and fill it with `Not activated -- `. -## Step 3: Workflow-only guardrails +## Step 3: Output contract -These guardrails are enforced at the workflow boundary. The skill -owns the review behavior; this step owns only the emission boundary. +- You may post **exactly one** comment for this entire panel run, and it + **must** be the final synthesized verdict from the **CEO** (after + arbitration). Sub-agent personas (Python Architect, CLI Logging + Expert, DevX UX Expert, Supply Chain Security Expert, OSS Growth + Hacker, Auth Expert when active) **do not** post comments — they + return their findings to the CEO, who synthesizes the single verdict. + When dispatching each sub-agent, instruct it explicitly: "do not post + any comment; return your findings to the orchestrator." +- Do not call the GitHub API directly. Write the comment via the + provided output channel; a downstream publisher posts it. -- Emit exactly **one** safe-output comment for this entire panel run. -- Do **not** call the GitHub API directly -- write only to the - `safe-outputs.add-comment` channel; the permission-isolated - downstream job publishes the comment to PR - #${{ github.event.pull_request.number || inputs.pr_number }}. -- ASCII only -- no emojis, no Unicode box-drawing (project encoding rule). +## Step 4: Emit the verdict -## Step 4: Emit the safe output - -Post the verdict by writing the comment body to the agent output channel. -The `safe-outputs.add-comment` job (capped at 1) will pick it up and -post it to PR #$PR. - -You do NOT call the GitHub API directly -- write the structured request to -the safe-outputs channel and gh-aw's permission-isolated downstream job -publishes the comment. +Write the CEO's final verdict comment body to the agent output channel. +The downstream publisher will post it to PR #$PR.