diff --git a/.github/workflows/agentic-observability-kit.lock.yml b/.github/workflows/agentic-observability-kit.lock.yml index 18e2108c1f4..9d781e4770c 100644 --- a/.github/workflows/agentic-observability-kit.lock.yml +++ b/.github/workflows/agentic-observability-kit.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"13d215de67e3180503449993078dc1d70a8fbf2f6c02f6f92166fc499625a661","strict":true,"agent_id":"copilot"} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/setup-go","sha":"4a3601121dd01d1626a1e23e37211e3254c1c06c","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"bcafcacb16a39f128d818304e6c9c0c18556b85f","version":"v7.1.0"},{"repo":"docker/setup-buildx-action","sha":"4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd","version":"v4"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.26"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.26"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.26"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.2.26"},{"image":"ghcr.io/github/github-mcp-server:v1.0.0"},{"image":"node:lts-alpine","digest":"sha256:01743339035a5c3c11a373cd7c83aeab6ed1457b55da6a69e014a95ac4e4700b","pinned_image":"node:lts-alpine@sha256:01743339035a5c3c11a373cd7c83aeab6ed1457b55da6a69e014a95ac4e4700b"}]} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"32bfcadb5379f7ddbeee6a5fb241af77127ddbce50852915bf390ea7eb71bdba","strict":true,"agent_id":"copilot"} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/setup-go","sha":"4a3601121dd01d1626a1e23e37211e3254c1c06c","version":"v6.4.0"},{"repo":"actions/setup-python","sha":"a309ff8b426b58ec0e2a45f0f869d46889d02405","version":"v6.2.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"bcafcacb16a39f128d818304e6c9c0c18556b85f","version":"v7.1.0"},{"repo":"docker/setup-buildx-action","sha":"4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd","version":"v4"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.26"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.26"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.26"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.2.26"},{"image":"ghcr.io/github/github-mcp-server:v1.0.0"},{"image":"node:lts-alpine","digest":"sha256:01743339035a5c3c11a373cd7c83aeab6ed1457b55da6a69e014a95ac4e4700b","pinned_image":"node:lts-alpine@sha256:01743339035a5c3c11a373cd7c83aeab6ed1457b55da6a69e014a95ac4e4700b"}]} # ___ _ _ # / _ \ | | (_) # | |_| | __ _ ___ _ __ | |_ _ ___ @@ -22,12 +22,13 @@ # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# Drop-in observability kit for repositories using agentic workflows +# Drop-in observability and portfolio review # # Resolved workflow manifest: # Imports: # - shared/daily-audit-discussion.md # - shared/reporting.md +# - shared/trending-charts-simple.md # # Secrets used: # - COPILOT_GITHUB_TOKEN @@ -36,10 +37,13 @@ # - GITHUB_TOKEN # # Custom actions used: +# - actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 +# - actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 # - actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 # - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 # - actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 # - actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 +# - actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 # - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 # - docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 # - docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 @@ -112,7 +116,7 @@ jobs: GH_AW_INFO_EXPERIMENTAL: "false" GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" GH_AW_INFO_STAGED: "false" - GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults","python"]' GH_AW_INFO_FIREWALL_ENABLED: "true" GH_AW_INFO_AWF_VERSION: "v0.25.26" GH_AW_INFO_AWMG_VERSION: "" @@ -179,21 +183,24 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_7d884fac12571b5f_EOF' + cat << 'GH_AW_PROMPT_983a611b9de05964_EOF' - GH_AW_PROMPT_7d884fac12571b5f_EOF + GH_AW_PROMPT_983a611b9de05964_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/agentic_workflows_guide.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/cache_memory_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_7d884fac12571b5f_EOF' + cat << 'GH_AW_PROMPT_983a611b9de05964_EOF' - Tools: create_issue, create_discussion, missing_tool, missing_data, noop + Tools: create_issue, create_discussion, upload_asset(max:4), missing_tool, missing_data, noop + + upload_asset: provide a file path; returns a URL; assets are published after the workflow completes (safeoutputs). - GH_AW_PROMPT_7d884fac12571b5f_EOF + GH_AW_PROMPT_983a611b9de05964_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_7d884fac12571b5f_EOF' + cat << 'GH_AW_PROMPT_983a611b9de05964_EOF' The following GitHub context information is available for this workflow: {{#if __GH_AW_GITHUB_ACTOR__ }} @@ -222,13 +229,14 @@ jobs: {{/if}} - GH_AW_PROMPT_7d884fac12571b5f_EOF + GH_AW_PROMPT_983a611b9de05964_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_7d884fac12571b5f_EOF' + cat << 'GH_AW_PROMPT_983a611b9de05964_EOF' + {{#runtime-import .github/workflows/shared/trending-charts-simple.md}} {{#runtime-import .github/workflows/shared/reporting.md}} {{#runtime-import .github/workflows/agentic-observability-kit.md}} - GH_AW_PROMPT_7d884fac12571b5f_EOF + GH_AW_PROMPT_983a611b9de05964_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 @@ -244,6 +252,9 @@ jobs: uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ALLOWED_EXTENSIONS: '' + GH_AW_CACHE_DESCRIPTION: '' + GH_AW_CACHE_DIR: '/tmp/gh-aw/cache-memory/' GH_AW_GITHUB_ACTOR: ${{ github.actor }} GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} @@ -264,6 +275,9 @@ jobs: return await substitutePlaceholders({ file: process.env.GH_AW_PROMPT, substitutions: { + GH_AW_ALLOWED_EXTENSIONS: process.env.GH_AW_ALLOWED_EXTENSIONS, + GH_AW_CACHE_DESCRIPTION: process.env.GH_AW_CACHE_DESCRIPTION, + GH_AW_CACHE_DIR: process.env.GH_AW_CACHE_DIR, GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, @@ -311,9 +325,9 @@ jobs: group: "gh-aw-copilot-${{ github.workflow }}" env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - GH_AW_ASSETS_ALLOWED_EXTS: "" - GH_AW_ASSETS_BRANCH: "" - GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_ASSETS_ALLOWED_EXTS: ".png,.svg" + GH_AW_ASSETS_BRANCH: "assets/${{ github.workflow }}" + GH_AW_ASSETS_MAX_SIZE_KB: 10240 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs GH_AW_WORKFLOW_ID_SANITIZED: agenticobservabilitykit outputs: @@ -385,12 +399,44 @@ jobs: tags: localhost/gh-aw:dev build-args: | BINARY=dist/gh-aw-linux-amd64 + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' - name: Create gh-aw temp directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" - name: Configure gh CLI for GitHub Enterprise run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + - name: Setup Python environment + run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/venv ]; then\n python3 -m venv /tmp/gh-aw/venv\nfi\necho \"/tmp/gh-aw/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n" + - if: always() + name: Upload source files and data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: warn + name: trending-source-and-data + path: | + /tmp/gh-aw/python/*.py + /tmp/gh-aw/python/data/* + retention-days: 30 + + # Cache memory file share configuration from frontmatter processed below + - name: Create cache-memory directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" + - name: Restore cache-memory file share data + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: memory-none-nopolicy-trending-data-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + restore-keys: | + memory-none-nopolicy-trending-data-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- + - name: Setup cache-memory git repository + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + GH_AW_MIN_INTEGRITY: none + run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} @@ -461,20 +507,23 @@ jobs: exit 1 fi - name: Write Safe Outputs Config + env: + GITHUB_WORKFLOW: ${{ github.workflow }} run: | mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_72c702bc45e03c43_EOF' - {"create_discussion":{"category":"audits","close_older_discussions":true,"expires":168,"fallback_to_issue":true,"max":1,"title_prefix":"[observability] "},"create_issue":{"close_older_issues":true,"labels":["agentics","warning","observability"],"max":1,"title_prefix":"[observability escalation] "},"create_report_incomplete_issue":{},"mentions":{"enabled":false},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_72c702bc45e03c43_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << GH_AW_SAFE_OUTPUTS_CONFIG_be7f952a9f7de614_EOF + {"create_discussion":{"category":"audits","close_older_discussions":true,"expires":168,"fallback_to_issue":true,"max":1,"title_prefix":"[observability] "},"create_issue":{"close_older_issues":true,"labels":["agentics","warning","observability"],"max":1,"title_prefix":"[observability escalation] "},"create_report_incomplete_issue":{},"mentions":{"enabled":false},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{},"upload_asset":{"allowed-exts":[".png",".svg"],"branch":"assets/${GITHUB_WORKFLOW}","max":4,"max-size":10240}} + GH_AW_SAFE_OUTPUTS_CONFIG_be7f952a9f7de614_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | { "description_suffixes": { "create_discussion": " CONSTRAINTS: Maximum 1 discussion(s) can be created. Title will be prefixed with \"[observability] \". Discussions will be created in category \"audits\".", - "create_issue": " CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[observability escalation] \". Labels [\"agentics\" \"warning\" \"observability\"] will be automatically added." + "create_issue": " CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[observability escalation] \". Labels [\"agentics\" \"warning\" \"observability\"] will be automatically added.", + "upload_asset": " CONSTRAINTS: Maximum 4 asset(s) can be uploaded. Maximum file size: 10240KB. Allowed file extensions: [.png .svg]." }, "repo_params": {}, "dynamic_tools": [] @@ -612,6 +661,15 @@ jobs: "maxLength": 1024 } } + }, + "upload_asset": { + "defaultMax": 10, + "fields": { + "path": { + "required": true, + "type": "string" + } + } } } uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 @@ -664,6 +722,9 @@ jobs: - name: Start MCP Gateway id: start-mcp-gateway env: + GH_AW_ASSETS_ALLOWED_EXTS: ${{ env.GH_AW_ASSETS_ALLOWED_EXTS }} + GH_AW_ASSETS_BRANCH: ${{ env.GH_AW_ASSETS_BRANCH }} + GH_AW_ASSETS_MAX_SIZE_KB: ${{ env.GH_AW_ASSETS_MAX_SIZE_KB }} GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} @@ -696,7 +757,7 @@ jobs: mkdir -p /home/runner/.copilot GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_6fa9cdf4c60ca0cb_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_4d8006432db968c9_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "agenticworkflows": { @@ -756,7 +817,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_6fa9cdf4c60ca0cb_EOF + GH_AW_MCP_CONFIG_4d8006432db968c9_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -796,12 +857,15 @@ jobs: export GH_AW_NODE_BIN (umask 177 && touch /tmp/gh-aw/agent-stdio.log) # shellcheck disable=SC1003 - sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --image-tag 0.25.26 --skip-pull --enable-api-proxy \ - -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || echo node)"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_driver.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains '*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --image-tag 0.25.26 --skip-pull --enable-api-proxy \ + -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || echo node)"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_driver.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir /tmp/gh-aw/cache-memory/ --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_ASSETS_ALLOWED_EXTS: ".png,.svg" + GH_AW_ASSETS_BRANCH: "assets/${{ github.workflow }}" + GH_AW_ASSETS_MAX_SIZE_KB: 10240 GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt @@ -883,7 +947,7 @@ jobs: uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 env: GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_DOMAINS: "*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GH_AW_ALLOWED_GITHUB_REFS: "" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} @@ -945,6 +1009,26 @@ jobs: if [ ! -f /tmp/gh-aw/agent_output.json ]; then echo '{"items":[]}' > /tmp/gh-aw/agent_output.json fi + - name: Commit cache-memory changes + if: always() + env: + GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory + run: bash "${RUNNER_TEMP}/gh-aw/actions/commit_cache_memory_git.sh" + - name: Upload cache-memory data as artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: cache-memory + path: /tmp/gh-aw/cache-memory + # Upload safe-outputs assets for upload_assets job + - name: Upload Safe Outputs Assets + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-assets + path: /tmp/gh-aw/safeoutputs/assets/ + retention-days: 1 + if-no-files-found: ignore - name: Upload agent artifacts if: always() continue-on-error: true @@ -974,6 +1058,8 @@ jobs: - agent - detection - safe_outputs + - update_cache_memory + - upload_assets if: > always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || needs.activation.outputs.stale_lock_file_failed == 'true') @@ -1211,7 +1297,7 @@ jobs: uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 env: WORKFLOW_NAME: "Agentic Observability Kit" - WORKFLOW_DESCRIPTION: "Drop-in observability kit for repositories using agentic workflows" + WORKFLOW_DESCRIPTION: "Drop-in observability and portfolio review" HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | @@ -1362,10 +1448,10 @@ jobs: uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_DOMAINS: "*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_discussion\":{\"category\":\"audits\",\"close_older_discussions\":true,\"expires\":168,\"fallback_to_issue\":true,\"max\":1,\"title_prefix\":\"[observability] \"},\"create_issue\":{\"close_older_issues\":true,\"labels\":[\"agentics\",\"warning\",\"observability\"],\"max\":1,\"title_prefix\":\"[observability escalation] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_discussion\":{\"category\":\"audits\",\"close_older_discussions\":true,\"expires\":168,\"fallback_to_issue\":true,\"max\":1,\"title_prefix\":\"[observability] \"},\"create_issue\":{\"close_older_issues\":true,\"labels\":[\"agentics\",\"warning\",\"observability\"],\"max\":1,\"title_prefix\":\"[observability escalation] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{},\"upload_asset\":{\"allowed-exts\":[\".png\",\".svg\"],\"branch\":\"assets/${{ github.workflow }}\",\"max\":4,\"max-size\":10240}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | @@ -1383,3 +1469,153 @@ jobs: /tmp/gh-aw/temporary-id-map.json if-no-files-found: ignore + update_cache_memory: + needs: + - activation + - agent + - detection + if: > + always() && (needs.detection.result == 'success' || needs.detection.result == 'skipped') && + needs.agent.result == 'success' + runs-on: ubuntu-slim + permissions: + contents: read + env: + GH_AW_WORKFLOW_ID_SANITIZED: agenticobservabilitykit + steps: + - name: Checkout actions folder + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: github/gh-aw + sparse-checkout: | + actions + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + - name: Download cache-memory artifact (default) + id: download_cache_default + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: cache-memory + path: /tmp/gh-aw/cache-memory + - name: Check if cache-memory folder has content (default) + id: check_cache_default + shell: bash + run: | + if [ -d "/tmp/gh-aw/cache-memory" ] && [ "$(ls -A /tmp/gh-aw/cache-memory 2>/dev/null)" ]; then + echo "has_content=true" >> "$GITHUB_OUTPUT" + else + echo "has_content=false" >> "$GITHUB_OUTPUT" + fi + - name: Save cache-memory to cache (default) + if: steps.check_cache_default.outputs.has_content == 'true' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: memory-none-nopolicy-trending-data-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} + path: /tmp/gh-aw/cache-memory + + upload_assets: + needs: + - activation + - agent + if: (!cancelled()) && needs.agent.result != 'skipped' && contains(needs.agent.outputs.output_types, 'upload_asset') + runs-on: ubuntu-slim + permissions: + contents: write + timeout-minutes: 10 + outputs: + branch_name: ${{ steps.upload_assets.outputs.branch_name }} + published_count: ${{ steps.upload_assets.outputs.published_count }} + steps: + - name: Checkout actions folder + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: github/gh-aw + sparse-checkout: | + actions + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Download assets + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: safe-outputs-assets + path: /tmp/gh-aw/safeoutputs/assets/ + - name: List downloaded asset files + continue-on-error: true + run: | + echo "Downloaded asset files:" + find /tmp/gh-aw/safeoutputs/assets/ -maxdepth 1 -ls + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Push assets + id: upload_assets + uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_ASSETS_BRANCH: "assets/${{ github.workflow }}" + GH_AW_ASSETS_MAX_SIZE_KB: 10240 + GH_AW_ASSETS_ALLOWED_EXTS: ".png,.svg" + GH_AW_WORKFLOW_NAME: "Agentic Observability Kit" + GH_AW_TRACKER_ID: "agentic-observability-kit" + GH_AW_ENGINE_ID: "copilot" + GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/upload_assets.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + persist-credentials: false + diff --git a/.github/workflows/agentic-observability-kit.md b/.github/workflows/agentic-observability-kit.md index ee827ef2acc..2f9e1fc4120 100644 --- a/.github/workflows/agentic-observability-kit.md +++ b/.github/workflows/agentic-observability-kit.md @@ -1,5 +1,5 @@ --- -description: Drop-in observability kit for repositories using agentic workflows +description: Drop-in observability and portfolio review on: schedule: weekly on monday around 08:00 workflow_dispatch: @@ -26,6 +26,9 @@ safe-outputs: labels: [agentics, warning, observability] close-older-issues: true max: 1 + upload-asset: + max: 4 + allowed-exts: [.png, .svg] noop: report-as-issue: false timeout-minutes: 30 @@ -34,6 +37,7 @@ imports: with: title-prefix: "[observability] " expires: 7d + - shared/trending-charts-simple.md - shared/reporting.md features: mcp-cli: true @@ -44,25 +48,33 @@ You are an agentic workflow observability analyst. Produce one executive report ## Mission -Review recent agentic workflow runs and surface the signals that matter operationally: +Review recent agentic workflow runs and surface the signals that matter operationally. Also include an evidence-based repository portfolio review so maintainers can spot low-value or overlapping workflows without running a separate workflow. + +The operational review remains primary. The portfolio review is a secondary appendix for repository maintainers, not a full organization-level governance exercise. + +Surface these signals: 1. Repeated drift away from a successful baseline 2. Weak control patterns such as new write posture, new MCP failures, or more blocked requests 3. Resource-heavy runs that are expensive for the domain they serve 4. Stable but low-value agentic runs that may be better as deterministic automation 5. Delegated workflows that lost continuity or are no longer behaving like a consistent cohort +6. Repository-local portfolio opportunities such as overlapping workflows, stale workflows, or high-cost workflows whose current usage does not justify their footprint Always create a discussion with the full report. Create an escalation issue only when repeated, actionable problems need durable owner follow-up. ## Data Collection Rules - Use the `agentic-workflows` MCP tool, not shell commands. -- Start with the `logs` tool over the last 14 days. +- Start with the `logs` tool over the last 30 days. - Leave `workflow_name` empty so you analyze the full repository. -- Use `count` large enough to cover the repository, typically `300`. +- Use `count` large enough to cover the repository, typically `300` to `500`. - Use the `audit` tool only for up to 3 runs that need deeper inspection. +- Use the `github` tool only for targeted inspection of workflow files when schedule, trigger, or overlap questions need confirmation. - If there are very few runs, still produce a report and explain the limitation. +Use the 30-day window for cost, repetition, and repository portfolio judgments. In the visible summary, emphasize the most recent 14 days when describing current regressions or urgent owner action. + ## Deterministic Episode Model The logs JSON now includes deterministic lineage fields: @@ -142,7 +154,7 @@ Treat these values as the canonical signals for reporting. ## Reporting Model -The discussion must stay concise and operator-friendly. +The discussion must stay concise and operator-friendly, but it should also provide immediate visual understanding. ### Visible Summary @@ -152,7 +164,9 @@ Keep these sections visible: 2. `### Key Metrics` 3. `### Highest Risk Episodes` 4. `### Episode Regressions` -5. `### Recommended Actions` +5. `### Visual Diagnostics` +6. `### Portfolio Opportunities` +7. `### Recommended Actions` Keep each visible section compact. Prefer short numeric summaries, 1-line judgments, and only the highest-value episodes. @@ -167,11 +181,271 @@ Include small numeric summaries such as: - workflows with repeated `overkill_for_agentic` - workflows with `partially_reducible` or `model_downgrade_available` assessments - workflows whose comparisons mostly fell back to `latest_success` +- workflows that look stale, overlapping, or weakly justified at repository scope ### Details Put detailed per-workflow breakdowns inside `
` blocks. +The `### Portfolio Opportunities` section should stay short. It is a repository-maintainer appendix, not a full workflow inventory. + +### Visual Report Form + +Generate exactly 4 high-quality charts using Python, pandas, matplotlib, and seaborn. Save them to `/tmp/gh-aw/python/charts/`, upload them with `upload_asset`, and embed them in the discussion. + +The charts should feel analytical and scientific. Favor phase-space views, heatmaps, frontier plots, and portfolio maps over decorative graphics. + +Use these concrete derived metrics: + +- `episode_risk_score` = weighted sum of `risky_node_count`, `poor_control_node_count`, `mcp_failure_count`, `blocked_request_count`, `new_mcp_failure_run_count`, and `blocked_request_increase_run_count`, with an additional boost when `escalation_eligible == true` +- `workflow_instability_score` = normalized combination of risky run rate, poor-control assessment rate, resource-heavy assessment rate, latest-success fallback rate, and blocked-request incidence +- `workflow_value_proxy` = normalized combination of successful recent usage, cohort stability, repeat use, and absence of overkill or partially reducible signals +- `workflow_overlap_score(a,b)` = blended similarity using task domain, workflow name or token overlap, trigger or schedule similarity from workflow files when inspected, behavioral fingerprint similarity, and common recommendation or assessment patterns in recent runs + +Use outcome-adjusted efficiency whenever the data supports it: + +- prefer `cost per successful run` over raw cost alone +- prefer `effective tokens per successful run` over raw tokens alone +- track tool or turn overhead as `median tool calls`, `unique tools`, and `turns` per successful run when possible +- treat a lower-cost workflow as better only when success and control quality remain comparable +- if `estimated_cost` is missing or zero for most workflows in the sample, switch the primary efficiency axis to `effective tokens per successful run` and state that choice explicitly in the report + +Required charts: + +1. **Episode Risk-Cost Frontier** + + Purpose: identify which execution chains are both expensive and risky. + + Build a scatter or bubble chart of the highest-cost or highest-risk episodes. + + - x-axis: `episodes[].total_estimated_cost` + - y-axis: `episode_risk_score` + - point size: `episodes[].total_runs` + - point color: `episodes[].primary_workflow` or task domain + - annotate the top outliers on the Pareto frontier + - save to `/tmp/gh-aw/python/charts/episode_risk_cost_frontier.png` + + Use this chart to answer: which episodes need immediate optimization or escalation because they combine cost with instability? + +2. **Workflow Stability Matrix** + + Purpose: identify repeat offenders and stable outliers. + + Build a workflow-by-metric heatmap where rows are workflows and columns are: + + - risky run rate + - latest-success fallback rate + - resource-heavy assessment rate + - poor-control assessment rate + - blocked-request incidence + - MCP failure incidence + + Sort rows by `workflow_instability_score` descending. + + Save to `/tmp/gh-aw/python/charts/workflow_stability_matrix.png`. + + Use this chart to answer: which workflows have chronic control or stability problems, and which ones are noisy only in one dimension? + +3. **Repository Portfolio Map** + + Purpose: separate workflows that should be kept, optimized, simplified, or reviewed. + + Build a scatter plot where each point is a workflow. + + - x-axis: recent cost, preferably `estimated_cost`, with effective tokens as fallback + - y-axis: `workflow_value_proxy` + - point size: recent run count + - point color: dominant recommendation bucket such as `keep`, `optimize`, `simplify`, `review` + + Add quadrant labels: + + - high value, low cost = `keep` + - high value, high cost = `optimize` + - low value, low cost = `simplify` + - low value, high cost = `review` + + Annotate the most extreme workflows and save to `/tmp/gh-aw/python/charts/repository_portfolio_map.png`. + + Use this chart to answer: which workflows deserve investment, which should be trimmed, and which demand a maintainer decision? + +4. **Workflow Overlap Matrix** + + Purpose: approximate portfolio overlap in a way that scales better than a literal Venn diagram. + + Build a symmetric workflow-by-workflow heatmap or clustered similarity matrix using `workflow_overlap_score(a,b)`. + + - use only the most relevant workflows to keep the matrix readable + - cluster similar workflows together if possible + - annotate the strongest non-trivial overlaps + - save to `/tmp/gh-aw/python/charts/workflow_overlap_matrix.png` + + Use this chart to answer: which workflows may be solving the same problem, sharing the same trigger space, or creating consolidation candidates? + +### Domain-Specific Reading Rules + +Interpret the same signals differently by task domain. + +- `triage`, `repo_maintenance`, and `issue_response`: these are the strongest candidates for deterministic replacement, smaller models, and narrow-tool routing. Penalize broad tool use, exploratory behavior, and repeated overkill more heavily here. +- `research`: tolerate broader tool breadth and exploratory behavior, but still penalize repeated high cost when it does not produce stable or differentiated value. Treat `partially_reducible` as especially important here because data gathering often belongs in deterministic pre-steps. +- `code_fix`: judge cost together with stability and successful actuation. Higher cost may be justified when write actions are intentional and controlled. Penalize blocked requests, poor control, and unstable write behavior more than raw cost alone. +- `release_ops`: prioritize reliability, repeatability, and low control friction. Moderate cost may be acceptable, but instability and fallback-heavy behavior are strong negatives. +- `delegated_automation`: prefer episode-level reading over workflow-level reading. Do not penalize a delegated worker for local cost or risk without checking the enclosing episode. +- `general_automation`: stay conservative and be explicit about uncertainty. + +When comparing workflows for portfolio decisions, compare workflows within similar task domains first. Avoid direct value claims between unlike domains unless the evidence is very strong. + +If task-domain coverage is weak, use a secondary grouping pass. + +- when most sampled runs collapse into `general_automation`, cluster workflows by behavior fingerprint and workflow family instead of pretending the domain split is precise +- use execution style, resource profile, tool breadth, dispatch mode, and workflow-name family as the fallback comparison frame +- say clearly when the portfolio view is behavior-clustered rather than domain-clustered + +Chart quality requirements: + +- 300 DPI minimum +- 12x7 inch figures unless the chart needs a square heatmap layout +- seaborn whitegrid or similarly clean scientific styling +- clear legends, axis labels, and direct annotations for outliers +- no chart should exist only for decoration; each one must support a decision in the report + +If the data is too sparse for one of these charts, generate a simpler fallback while preserving the same filename and explain the limitation briefly in the report. + +Embed the charts in `### Visual Diagnostics` with short, decision-oriented interpretation under each chart. + +### Visual Diagnostics Report Template + +Use this exact report form inside the discussion: + +```markdown +### Visual Diagnostics + +#### 1. Episode Risk-Cost Frontier +![Episode Risk-Cost Frontier](URL_FROM_UPLOAD_ASSET_1) + +Decision: +[One sentence naming the frontier episodes and the immediate implication.] + +Why it matters: +[One or two sentences explaining whether the problem is cost-heavy, risk-heavy, or both.] + +#### 2. Workflow Stability Matrix +![Workflow Stability Matrix](URL_FROM_UPLOAD_ASSET_2) + +Decision: +[One sentence naming the least stable workflows or the most concentrated failure mode.] + +Why it matters: +[One or two sentences explaining whether the repository has broad instability or a few repeat offenders.] + +#### 3. Repository Portfolio Map +![Repository Portfolio Map](URL_FROM_UPLOAD_ASSET_3) + +Decision: +[One sentence naming the workflows that belong in keep, optimize, simplify, or review.] + +Why it matters: +[One or two sentences explaining the portfolio tradeoff visible in the quadrants.] + +#### 4. Workflow Overlap Matrix +![Workflow Overlap Matrix](URL_FROM_UPLOAD_ASSET_4) + +Decision: +[One sentence naming the strongest consolidation or overlap candidates.] + +Why it matters: +[One or two sentences explaining whether the overlap appears real, weak, or uncertain.] +``` + +Each chart must end in a decision, not only an observation. + +Across the full report, prefer statements like `high cost but justified`, `cheap but unstable`, `expensive and dominated`, or `agentic overkill for this domain` over generic labels like `good` or `bad`. + +### Plot Construction Notes + +Write a Python script to `/tmp/gh-aw/python/agentic_observability_plots.py` and run it. + +Use pandas DataFrames for both episode-level and workflow-level aggregation. + +Recommended construction flow: + +1. Build an `episodes_df` with one row per episode using `episodes[]`. +2. Build a `runs_df` with one row per run using `.runs[]`. +3. Build a `workflow_df` by grouping `runs_df` by workflow name. +4. Derive chart-specific metrics from those three tables. + +Use logic of this form: + +```python +import numpy as np +import pandas as pd + +def normalize(series): + if series.empty: + return series + spread = series.max() - series.min() + if spread == 0: + return pd.Series([0.0] * len(series), index=series.index) + return (series - series.min()) / spread + +episodes_df["episode_risk_score"] = ( + 1.0 * episodes_df["risky_node_count"].fillna(0) + + 1.2 * episodes_df["poor_control_node_count"].fillna(0) + + 1.2 * episodes_df["mcp_failure_count"].fillna(0) + + 1.0 * episodes_df["blocked_request_count"].fillna(0) + + 1.4 * episodes_df["new_mcp_failure_run_count"].fillna(0) + + 1.4 * episodes_df["blocked_request_increase_run_count"].fillna(0) + + 2.0 * episodes_df["escalation_eligible"].fillna(False).astype(int) +) + +workflow_df["cost_per_successful_run"] = ( + workflow_df["estimated_cost_sum"] / + workflow_df["successful_runs"].replace(0, np.nan) +) + +workflow_df["workflow_instability_score"] = ( + 0.25 * normalize(workflow_df["risky_run_rate"].fillna(0)) + + 0.20 * normalize(workflow_df["poor_control_rate"].fillna(0)) + + 0.20 * normalize(workflow_df["resource_heavy_rate"].fillna(0)) + + 0.15 * normalize(workflow_df["latest_success_fallback_rate"].fillna(0)) + + 0.10 * normalize(workflow_df["blocked_request_rate"].fillna(0)) + + 0.10 * normalize(workflow_df["mcp_failure_rate"].fillna(0)) +) + +workflow_df["workflow_value_proxy"] = ( + 0.35 * normalize(workflow_df["successful_runs"].fillna(0)) + + 0.25 * (1.0 - normalize(workflow_df["workflow_instability_score"].fillna(0))) + + 0.20 * normalize(workflow_df["repeat_use_score"].fillna(0)) + + 0.20 * (1.0 - normalize(workflow_df["overkill_or_reduction_signal_rate"].fillna(0))) +) +``` + +For the overlap matrix, do not overclaim. Use a blended similarity score and label it as approximate when needed. + +Use logic of this form: + +```python +def workflow_overlap_score(row_a, row_b): + score = 0.0 + if row_a["task_domain"] == row_b["task_domain"]: + score += 0.30 + if row_a["schedule_or_trigger_family"] == row_b["schedule_or_trigger_family"]: + score += 0.25 + if row_a["behavior_cluster"] == row_b["behavior_cluster"]: + score += 0.20 + score += 0.15 * row_a["name_similarity_to"][row_b["workflow_name"]] + score += 0.10 * row_a["assessment_similarity_to"][row_b["workflow_name"]] + return min(score, 1.0) +``` + +Fallback guidance: + +- if there are too few episodes for a meaningful frontier, plot only the top workflows by cost and annotate the limitation +- if the overlap matrix is too sparse, restrict it to the top 8 to 12 workflows by recent cost, run volume, or instability +- if successful-run counts are too low, fall back from cost-per-successful-run to recent cost with a clear caveat +- if `estimated_cost` is sparse across the repository, fall back from recent cost to `effective_tokens` or `effective tokens per successful run` +- if more than half of sampled runs are `general_automation`, add a note that domain confidence is low and use behavior clusters for portfolio comparisons +- if `partially_reducible` is common across the sample, do not demote workflows on that signal alone; require repetition, high token overhead, or supporting instability before labeling a workflow `review` + ### What Good Reporting Looks Like For each highlighted episode or workflow, explain: @@ -187,6 +461,29 @@ For each highlighted episode or workflow, explain: Do not turn the visible summary into a full inventory. Push secondary detail into `
` blocks. +## Evidence-Based Repository Portfolio Review + +Include an evidence-based repository portfolio review in the discussion. Keep it secondary to the operational review and bounded to what can be supported by recent run data plus targeted workflow-file inspection. + +Good repository-level portfolio questions include: + +- Which workflows appear repeatedly overkill for their task domain? +- Which workflows look expensive relative to their recent value or stability? +- Which workflows may overlap in purpose, trigger pattern, or schedule? +- Which scheduled workflows look underused or weakly justified by recent activity? + +When making portfolio judgments: + +- Prefer evidence from recent run history, repeated assessments, and episode patterns. +- Use targeted workflow-file inspection only to confirm schedule or trigger overlap. +- Be explicit about uncertainty when recent data is too thin. +- Do not claim repository-local overlap with high confidence unless names, task domains, run patterns, or workflow definitions materially support it. +- Do not attempt organization-wide governance or replacement planning from this repository-level report. + +Treat these as optimization and cleanup opportunities unless they also cross the operational escalation thresholds. + +When the evidence supports it, use the visual charts to strengthen portfolio observations. For example, use the overlap matrix to support possible consolidation, or the portfolio map to justify why a workflow belongs in `optimize` rather than `retire`. + ## Escalation Thresholds Use the discussion as the complete source of truth for all qualifying workflows and episodes. Prefer episode-level escalation when `episodes[].escalation_eligible == true`. Only fall back to workflow-level counting when episode data is missing or clearly incomplete. @@ -220,6 +517,8 @@ Do not create issues for these by default. Report them in the discussion unless - repeated `overkill_for_agentic` - workflows that are consistently `lean`, `directed`, and `narrow` - workflows that are always compared using `latest_success` instead of `cohort_match` +- workflows whose recent activity or cost makes them look weakly justified at repository scope +- workflows that appear to overlap and could plausibly be consolidated These are portfolio cleanup opportunities, not immediate incidents. @@ -245,7 +544,9 @@ Always create one discussion that includes: - all workflows that crossed the escalation thresholds - the workflows with the clearest repeated risk - the most common assessment kinds +- 4 embedded charts in a `### Visual Diagnostics` section - a short list of deterministic candidates +- a short repository-level portfolio section covering stale, overlapping, or weakly justified workflows when the evidence supports it - a short list of workflows that need owner attention now The discussion should cover all qualifying workflows even when no escalation issue is created. diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml deleted file mode 100644 index 00902ec91d7..00000000000 --- a/.github/workflows/portfolio-analyst.lock.yml +++ /dev/null @@ -1,1624 +0,0 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"2b959291116f6dfd78a87dfb3f7d3f0109835180d0471ba6c0c35409866ae36d","strict":true,"agent_id":"copilot"} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/setup-go","sha":"4a3601121dd01d1626a1e23e37211e3254c1c06c","version":"v6.4.0"},{"repo":"actions/setup-python","sha":"a309ff8b426b58ec0e2a45f0f869d46889d02405","version":"v6.2.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"bcafcacb16a39f128d818304e6c9c0c18556b85f","version":"v7.1.0"},{"repo":"docker/setup-buildx-action","sha":"4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd","version":"v4"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.26"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.26"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.26"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.2.26"},{"image":"ghcr.io/github/github-mcp-server:v1.0.0"},{"image":"node:lts-alpine","digest":"sha256:01743339035a5c3c11a373cd7c83aeab6ed1457b55da6a69e014a95ac4e4700b","pinned_image":"node:lts-alpine@sha256:01743339035a5c3c11a373cd7c83aeab6ed1457b55da6a69e014a95ac4e4700b"}]} -# ___ _ _ -# / _ \ | | (_) -# | |_| | __ _ ___ _ __ | |_ _ ___ -# | _ |/ _` |/ _ \ '_ \| __| |/ __| -# | | | | (_| | __/ | | | |_| | (__ -# \_| |_/\__, |\___|_| |_|\__|_|\___| -# __/ | -# _ _ |___/ -# | | | | / _| | -# | | | | ___ _ __ _ __| |_| | _____ ____ -# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| -# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ -# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ -# -# This file was automatically generated by gh-aw. DO NOT EDIT. -# -# To update this file, edit the corresponding .md file and run: -# gh aw compile -# Not all edits will cause changes to this file. -# -# For more information: https://github.github.com/gh-aw/introduction/overview/ -# -# Weekly portfolio analyst that identifies cost reduction opportunities (20%+) while improving workflow reliability -# -# Resolved workflow manifest: -# Imports: -# - shared/daily-audit-discussion.md -# - shared/jqschema.md -# - shared/reporting.md -# - shared/trending-charts-simple.md -# Includes: -# - shared/noop-reminder.md -# -# Secrets used: -# - COPILOT_GITHUB_TOKEN -# - GH_AW_GITHUB_MCP_SERVER_TOKEN -# - GH_AW_GITHUB_TOKEN -# - GITHUB_TOKEN -# -# Custom actions used: -# - actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 -# - actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 -# - actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 -# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 -# - actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 -# - actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 -# - actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 -# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 -# - docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 -# - docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 -# -# Container images used: -# - ghcr.io/github/gh-aw-firewall/agent:0.25.26 -# - ghcr.io/github/gh-aw-firewall/api-proxy:0.25.26 -# - ghcr.io/github/gh-aw-firewall/squid:0.25.26 -# - ghcr.io/github/gh-aw-mcpg:v0.2.26 -# - ghcr.io/github/github-mcp-server:v1.0.0 -# - node:lts-alpine@sha256:01743339035a5c3c11a373cd7c83aeab6ed1457b55da6a69e014a95ac4e4700b - -name: "Automated Portfolio Analyst" -"on": - schedule: - - cron: "50 8 * * 1" - # Friendly format: weekly on monday around 09:00 (scattered) - workflow_dispatch: - inputs: - aw_context: - default: "" - description: Agent caller context (used internally by Agentic Workflows). - required: false - type: string - -permissions: {} - -concurrency: - group: "gh-aw-${{ github.workflow }}" - -run-name: "Automated Portfolio Analyst" - -jobs: - activation: - runs-on: ubuntu-slim - permissions: - actions: read - contents: read - outputs: - comment_id: "" - comment_repo: "" - lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} - model: ${{ steps.generate_aw_info.outputs.model }} - secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} - setup-trace-id: ${{ steps.setup.outputs.trace-id }} - stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} - steps: - - name: Checkout actions folder - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - repository: github/gh-aw - sparse-checkout: | - actions - persist-credentials: false - - name: Setup Scripts - id: setup - uses: ./actions/setup - with: - destination: ${{ runner.temp }}/gh-aw/actions - job-name: ${{ github.job }} - - name: Generate agentic run info - id: generate_aw_info - env: - GH_AW_INFO_ENGINE_ID: "copilot" - GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" - GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'auto' }} - GH_AW_INFO_VERSION: "1.0.21" - GH_AW_INFO_AGENT_VERSION: "1.0.21" - GH_AW_INFO_WORKFLOW_NAME: "Automated Portfolio Analyst" - GH_AW_INFO_EXPERIMENTAL: "false" - GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" - GH_AW_INFO_STAGED: "false" - GH_AW_INFO_ALLOWED_DOMAINS: '["defaults","python"]' - GH_AW_INFO_FIREWALL_ENABLED: "true" - GH_AW_INFO_AWF_VERSION: "v0.25.26" - GH_AW_INFO_AWMG_VERSION: "" - GH_AW_INFO_FIREWALL_TYPE: "squid" - GH_AW_COMPILED_STRICT: "true" - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); - await main(core, context); - - name: Validate COPILOT_GITHUB_TOKEN secret - id: validate-secret - run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default - env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - - name: Checkout .github and .agents folders - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - sparse-checkout: | - .github - .agents - actions/setup - .claude - .codex - .crush - .gemini - .opencode - sparse-checkout-cone-mode: true - fetch-depth: 1 - - name: Save agent config folders for base branch restoration - env: - GH_AW_AGENT_FOLDERS: ".agents .claude .codex .crush .gemini .github .opencode" - GH_AW_AGENT_FILES: ".crush.json AGENTS.md CLAUDE.md GEMINI.md opencode.jsonc" - # poutine:ignore untrusted_checkout_exec - run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" - - name: Check workflow lock file - id: check-lock-file - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_WORKFLOW_FILE: "portfolio-analyst.lock.yml" - GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); - await main(); - - name: Create prompt with built-in context - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - # poutine:ignore untrusted_checkout_exec - run: | - bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" - { - cat << 'GH_AW_PROMPT_ab2c5d8e37806733_EOF' - - GH_AW_PROMPT_ab2c5d8e37806733_EOF - cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/agentic_workflows_guide.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/cache_memory_prompt.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_ab2c5d8e37806733_EOF' - - Tools: create_discussion, upload_asset(max:5), missing_tool, missing_data, noop - - upload_asset: provide a file path; returns a URL; assets are published after the workflow completes (safeoutputs). - - GH_AW_PROMPT_ab2c5d8e37806733_EOF - cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_ab2c5d8e37806733_EOF' - - The following GitHub context information is available for this workflow: - {{#if __GH_AW_GITHUB_ACTOR__ }} - - **actor**: __GH_AW_GITHUB_ACTOR__ - {{/if}} - {{#if __GH_AW_GITHUB_REPOSITORY__ }} - - **repository**: __GH_AW_GITHUB_REPOSITORY__ - {{/if}} - {{#if __GH_AW_GITHUB_WORKSPACE__ }} - - **workspace**: __GH_AW_GITHUB_WORKSPACE__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} - - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} - - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} - - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ - {{/if}} - {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} - - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ - {{/if}} - {{#if __GH_AW_GITHUB_RUN_ID__ }} - - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ - {{/if}} - - - GH_AW_PROMPT_ab2c5d8e37806733_EOF - cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_ab2c5d8e37806733_EOF' - - {{#runtime-import .github/workflows/shared/reporting.md}} - {{#runtime-import .github/workflows/shared/jqschema.md}} - {{#runtime-import .github/workflows/shared/trending-charts-simple.md}} - {{#runtime-import .github/workflows/portfolio-analyst.md}} - GH_AW_PROMPT_ab2c5d8e37806733_EOF - } > "$GH_AW_PROMPT" - - name: Interpolate variables and render templates - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); - await main(); - - name: Substitute placeholders - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_ALLOWED_EXTENSIONS: '' - GH_AW_CACHE_DESCRIPTION: '' - GH_AW_CACHE_DIR: '/tmp/gh-aw/cache-memory/' - GH_AW_GITHUB_ACTOR: ${{ github.actor }} - GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} - GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_MCP_CLI_SERVERS_LIST: "- `agenticworkflows` — run `agenticworkflows --help` to see available tools\n- `safeoutputs` — run `safeoutputs --help` to see available tools" - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - - const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); - - // Call the substitution function - return await substitutePlaceholders({ - file: process.env.GH_AW_PROMPT, - substitutions: { - GH_AW_ALLOWED_EXTENSIONS: process.env.GH_AW_ALLOWED_EXTENSIONS, - GH_AW_CACHE_DESCRIPTION: process.env.GH_AW_CACHE_DESCRIPTION, - GH_AW_CACHE_DIR: process.env.GH_AW_CACHE_DIR, - GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, - GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, - GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, - GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, - GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, - GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, - GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, - GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, - GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST - } - }); - - name: Validate prompt placeholders - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - # poutine:ignore untrusted_checkout_exec - run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" - - name: Print prompt - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - # poutine:ignore untrusted_checkout_exec - run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" - - name: Upload activation artifact - if: success() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: activation - path: | - /tmp/gh-aw/aw_info.json - /tmp/gh-aw/aw-prompts/prompt.txt - /tmp/gh-aw/github_rate_limits.jsonl - /tmp/gh-aw/base - if-no-files-found: ignore - retention-days: 1 - - agent: - needs: activation - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - issues: read - pull-requests: read - concurrency: - group: "gh-aw-copilot-${{ github.workflow }}" - env: - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - GH_AW_ASSETS_ALLOWED_EXTS: ".png,.jpg,.jpeg,.svg" - GH_AW_ASSETS_BRANCH: "assets/${{ github.workflow }}" - GH_AW_ASSETS_MAX_SIZE_KB: 10240 - GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs - GH_AW_WORKFLOW_ID_SANITIZED: portfolioanalyst - outputs: - agentic_engine_timeout: ${{ steps.detect-copilot-errors.outputs.agentic_engine_timeout || 'false' }} - checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} - effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} - has_patch: ${{ steps.collect_output.outputs.has_patch }} - inference_access_error: ${{ steps.detect-copilot-errors.outputs.inference_access_error || 'false' }} - mcp_policy_error: ${{ steps.detect-copilot-errors.outputs.mcp_policy_error || 'false' }} - model: ${{ needs.activation.outputs.model }} - model_not_supported_error: ${{ steps.detect-copilot-errors.outputs.model_not_supported_error || 'false' }} - output: ${{ steps.collect_output.outputs.output }} - output_types: ${{ steps.collect_output.outputs.output_types }} - setup-trace-id: ${{ steps.setup.outputs.trace-id }} - steps: - - name: Checkout actions folder - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - repository: github/gh-aw - sparse-checkout: | - actions - persist-credentials: false - - name: Setup Scripts - id: setup - uses: ./actions/setup - with: - destination: ${{ runner.temp }}/gh-aw/actions - job-name: ${{ github.job }} - trace-id: ${{ needs.activation.outputs.setup-trace-id }} - - name: Set runtime paths - id: set-runtime-paths - run: | - { - echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" - echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" - echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" - } >> "$GITHUB_OUTPUT" - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Go for CLI build - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - cache: true - - name: Build gh-aw CLI - run: | - echo "Building gh-aw CLI for linux/amd64..." - mkdir -p dist - VERSION=$(git describe --tags --always --dirty) - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ - -ldflags "-s -w -X main.version=${VERSION}" \ - -o dist/gh-aw-linux-amd64 \ - ./cmd/gh-aw - # Copy binary to root for direct execution in user-defined steps - cp dist/gh-aw-linux-amd64 ./gh-aw - chmod +x ./gh-aw - echo "✓ Built gh-aw CLI successfully" - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - - name: Build gh-aw Docker image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - context: . - platforms: linux/amd64 - push: false - load: true - tags: localhost/gh-aw:dev - build-args: | - BINARY=dist/gh-aw-linux-amd64 - - name: Setup Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.12' - - name: Create gh-aw temp directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" - - name: Configure gh CLI for GitHub Enterprise - run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" - env: - GH_TOKEN: ${{ github.token }} - - name: Setup jq utilities directory - run: "mkdir -p /tmp/gh-aw\ncat > /tmp/gh-aw/jqschema.sh << 'EOF'\n#!/usr/bin/env bash\n# jqschema.sh\njq -c '\ndef walk(f):\n . as $in |\n if type == \"object\" then\n reduce keys[] as $k ({}; . + {($k): ($in[$k] | walk(f))})\n elif type == \"array\" then\n if length == 0 then [] else [.[0] | walk(f)] end\n else\n type\n end;\nwalk(.)\n'\nEOF\nchmod +x /tmp/gh-aw/jqschema.sh\n" - - name: Setup Python environment - run: "mkdir -p /tmp/gh-aw/python/{data,charts,artifacts}\n# Create a virtual environment for proper package isolation (avoids --break-system-packages)\nif [ ! -d /tmp/gh-aw/venv ]; then\n python3 -m venv /tmp/gh-aw/venv\nfi\necho \"/tmp/gh-aw/venv/bin\" >> \"$GITHUB_PATH\"\n/tmp/gh-aw/venv/bin/pip install --quiet numpy pandas matplotlib seaborn scipy\n" - - if: always() - name: Upload source files and data - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: warn - name: trending-source-and-data - path: | - /tmp/gh-aw/python/*.py - /tmp/gh-aw/python/data/* - retention-days: 30 - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Install gh-aw CLI - run: | - if gh extension list | grep -q "github/gh-aw"; then - gh extension upgrade gh-aw || true - else - gh extension install github/gh-aw - fi - gh aw --version - - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Download logs from last 30 days - run: |- - mkdir -p /tmp/portfolio-logs - gh aw logs --start-date -30d -c 5000 -o /tmp/portfolio-logs --json > /tmp/portfolio-logs/summary.json - - # Cache memory file share configuration from frontmatter processed below - - name: Create cache-memory directory - run: bash "${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh" - - name: Restore cache-memory file share data - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - key: memory-none-nopolicy-trending-data-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - restore-keys: | - memory-none-nopolicy-trending-data-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}- - - name: Setup cache-memory git repository - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - GH_AW_MIN_INTEGRITY: none - run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" - - name: Configure Git credentials - env: - REPO_NAME: ${{ github.repository }} - SERVER_URL: ${{ github.server_url }} - GITHUB_TOKEN: ${{ github.token }} - run: | - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - git config --global am.keepcr true - # Re-authenticate git with GitHub token - SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" - echo "Git configured with standard GitHub Actions identity" - - name: Checkout PR branch - id: checkout-pr - if: | - github.event.pull_request || github.event.issue.pull_request - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - with: - github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs'); - await main(); - - name: Install GitHub Copilot CLI - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.21 - env: - GH_HOST: github.com - - name: Install AWF binary - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.26 - - name: Determine automatic lockdown mode for GitHub MCP Server - id: determine-automatic-lockdown - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} - GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} - with: - script: | - const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs'); - await determineAutomaticLockdown(github, context, core); - - name: Download container images - run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.26 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.26 ghcr.io/github/gh-aw-firewall/squid:0.25.26 ghcr.io/github/gh-aw-mcpg:v0.2.26 ghcr.io/github/github-mcp-server:v1.0.0 node:lts-alpine@sha256:01743339035a5c3c11a373cd7c83aeab6ed1457b55da6a69e014a95ac4e4700b - - name: Install gh-aw extension - env: - GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - run: | - # Check if gh-aw extension is already installed - if gh extension list | grep -q "github/gh-aw"; then - echo "gh-aw extension already installed, upgrading..." - gh extension upgrade gh-aw || true - else - echo "Installing gh-aw extension..." - gh extension install github/gh-aw - fi - gh aw --version - # Copy the gh-aw binary to ${RUNNER_TEMP}/gh-aw for MCP server containerization - mkdir -p "${RUNNER_TEMP}/gh-aw" - GH_AW_BIN=$(which gh-aw 2>/dev/null || find ~/.local/share/gh/extensions/gh-aw -name 'gh-aw' -type f 2>/dev/null | head -1) - if [ -n "$GH_AW_BIN" ] && [ -f "$GH_AW_BIN" ]; then - cp "$GH_AW_BIN" "${RUNNER_TEMP}/gh-aw/gh-aw" - chmod +x "${RUNNER_TEMP}/gh-aw/gh-aw" - echo "Copied gh-aw binary to ${RUNNER_TEMP}/gh-aw/gh-aw" - else - echo "::error::Failed to find gh-aw binary for MCP server" - exit 1 - fi - - name: Write Safe Outputs Config - env: - GITHUB_WORKFLOW: ${{ github.workflow }} - run: | - mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" - mkdir -p /tmp/gh-aw/safeoutputs - mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs/upload-artifacts" - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << GH_AW_SAFE_OUTPUTS_CONFIG_81498801d5bc1dfc_EOF - {"create_discussion":{"category":"audits","close_older_discussions":true,"expires":24,"fallback_to_issue":true,"max":1,"title_prefix":"[portfolio] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{},"upload_artifact":{"max-size-bytes":104857600,"max-uploads":3,"retention-days":30,"skip-archive":true},"upload_asset":{"allowed-exts":[".png",".jpg",".jpeg",".svg"],"branch":"assets/${GITHUB_WORKFLOW}","max":5,"max-size":10240}} - GH_AW_SAFE_OUTPUTS_CONFIG_81498801d5bc1dfc_EOF - - name: Write Safe Outputs Tools - env: - GH_AW_TOOLS_META_JSON: | - { - "description_suffixes": { - "create_discussion": " CONSTRAINTS: Maximum 1 discussion(s) can be created. Title will be prefixed with \"[portfolio] \". Discussions will be created in category \"audits\".", - "upload_asset": " CONSTRAINTS: Maximum 5 asset(s) can be uploaded. Maximum file size: 10240KB. Allowed file extensions: [.png .jpg .jpeg .svg]." - }, - "repo_params": {}, - "dynamic_tools": [] - } - GH_AW_VALIDATION_JSON: | - { - "create_discussion": { - "defaultMax": 1, - "fields": { - "body": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 65000 - }, - "category": { - "type": "string", - "sanitize": true, - "maxLength": 128 - }, - "repo": { - "type": "string", - "maxLength": 256 - }, - "title": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 128 - } - } - }, - "missing_data": { - "defaultMax": 20, - "fields": { - "alternatives": { - "type": "string", - "sanitize": true, - "maxLength": 256 - }, - "context": { - "type": "string", - "sanitize": true, - "maxLength": 256 - }, - "data_type": { - "type": "string", - "sanitize": true, - "maxLength": 128 - }, - "reason": { - "type": "string", - "sanitize": true, - "maxLength": 256 - } - } - }, - "missing_tool": { - "defaultMax": 20, - "fields": { - "alternatives": { - "type": "string", - "sanitize": true, - "maxLength": 512 - }, - "reason": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 256 - }, - "tool": { - "type": "string", - "sanitize": true, - "maxLength": 128 - } - } - }, - "noop": { - "defaultMax": 1, - "fields": { - "message": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 65000 - } - } - }, - "report_incomplete": { - "defaultMax": 5, - "fields": { - "details": { - "type": "string", - "sanitize": true, - "maxLength": 65000 - }, - "reason": { - "required": true, - "type": "string", - "sanitize": true, - "maxLength": 1024 - } - } - }, - "upload_asset": { - "defaultMax": 10, - "fields": { - "path": { - "required": true, - "type": "string" - } - } - } - } - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); - await main(); - - name: Generate Safe Outputs MCP Server Config - id: safe-outputs-config - run: | - # Generate a secure random API key (360 bits of entropy, 40+ chars) - # Mask immediately to prevent timing vulnerabilities - API_KEY=$(openssl rand -base64 45 | tr -d '/+=') - echo "::add-mask::${API_KEY}" - - PORT=3001 - - # Set outputs for next steps - { - echo "safe_outputs_api_key=${API_KEY}" - echo "safe_outputs_port=${PORT}" - } >> "$GITHUB_OUTPUT" - - echo "Safe Outputs MCP server will run on port ${PORT}" - - - name: Start Safe Outputs MCP HTTP Server - id: safe-outputs-start - env: - DEBUG: '*' - GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} - GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} - GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/tools.json - GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/config.json - GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs - run: | - # Environment variables are set above to prevent template injection - export DEBUG - export GH_AW_SAFE_OUTPUTS - export GH_AW_SAFE_OUTPUTS_PORT - export GH_AW_SAFE_OUTPUTS_API_KEY - export GH_AW_SAFE_OUTPUTS_TOOLS_PATH - export GH_AW_SAFE_OUTPUTS_CONFIG_PATH - export GH_AW_MCP_LOG_DIR - - bash "${RUNNER_TEMP}/gh-aw/actions/start_safe_outputs_server.sh" - - - name: Start MCP Gateway - id: start-mcp-gateway - env: - GH_AW_ASSETS_ALLOWED_EXTS: ${{ env.GH_AW_ASSETS_ALLOWED_EXTS }} - GH_AW_ASSETS_BRANCH: ${{ env.GH_AW_ASSETS_BRANCH }} - GH_AW_ASSETS_MAX_SIZE_KB: ${{ env.GH_AW_ASSETS_MAX_SIZE_KB }} - GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} - GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} - GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }} - GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }} - GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -eo pipefail - mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" - - # Export gateway environment variables for MCP config and gateway script - export MCP_GATEWAY_PORT="8080" - export MCP_GATEWAY_DOMAIN="host.docker.internal" - MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') - echo "::add-mask::${MCP_GATEWAY_API_KEY}" - export MCP_GATEWAY_API_KEY - export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" - mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" - export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" - export DEBUG="*" - - export GH_AW_ENGINE="copilot" - export GH_AW_MCP_CLI_SERVERS='["agenticworkflows"]' - echo 'GH_AW_MCP_CLI_SERVERS=["agenticworkflows"]' >> "$GITHUB_ENV" - MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') - MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') - DOCKER_SOCK_GID=$(stat -c '%g' /var/run/docker.sock 2>/dev/null || echo '0') - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.26' - - mkdir -p /home/runner/.copilot - GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_ec0ceddbc6b9bb44_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" - { - "mcpServers": { - "agenticworkflows": { - "type": "stdio", - "container": "localhost/gh-aw:dev", - "mounts": ["\${GITHUB_WORKSPACE}:\${GITHUB_WORKSPACE}:rw", "/tmp/gh-aw:/tmp/gh-aw:rw"], - "args": ["--network", "host", "-w", "\${GITHUB_WORKSPACE}"], - "env": { - "DEBUG": "*", - "GITHUB_TOKEN": "\${GITHUB_TOKEN}", - "GITHUB_ACTOR": "\${GITHUB_ACTOR}", - "GITHUB_REPOSITORY": "\${GITHUB_REPOSITORY}" - }, - "guard-policies": { - "write-sink": { - "accept": [ - "*" - ] - } - } - }, - "github": { - "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v1.0.0", - "env": { - "GITHUB_HOST": "\${GITHUB_SERVER_URL}", - "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", - "GITHUB_READ_ONLY": "1", - "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" - }, - "guard-policies": { - "allow-only": { - "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY", - "repos": "$GITHUB_MCP_GUARD_REPOS" - } - } - }, - "safeoutputs": { - "type": "http", - "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", - "headers": { - "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" - }, - "guard-policies": { - "write-sink": { - "accept": [ - "*" - ] - } - } - } - }, - "gateway": { - "port": $MCP_GATEWAY_PORT, - "domain": "${MCP_GATEWAY_DOMAIN}", - "apiKey": "${MCP_GATEWAY_API_KEY}", - "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" - } - } - GH_AW_MCP_CONFIG_ec0ceddbc6b9bb44_EOF - - name: Mount MCP servers as CLIs - id: mount-mcp-clis - continue-on-error: true - env: - MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} - MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} - MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io); - const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs'); - await main(); - - name: Download activation artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: activation - path: /tmp/gh-aw - - name: Restore agent config folders from base branch - if: steps.checkout-pr.outcome == 'success' - env: - GH_AW_AGENT_FOLDERS: ".agents .claude .codex .crush .gemini .github .opencode" - GH_AW_AGENT_FILES: ".crush.json AGENTS.md CLAUDE.md GEMINI.md opencode.jsonc" - run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh" - - name: Clean git credentials - continue-on-error: true - run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh" - - name: Execute GitHub Copilot CLI - id: agentic_execution - # Copilot CLI tool arguments (sorted): - timeout-minutes: 20 - run: | - set -o pipefail - touch /tmp/gh-aw/agent-step-summary.md - GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) - export GH_AW_NODE_BIN - (umask 177 && touch /tmp/gh-aw/agent-stdio.log) - # shellcheck disable=SC1003 - sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw/safeoutputs/upload-artifacts:${RUNNER_TEMP}/gh-aw/safeoutputs/upload-artifacts:rw" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains '*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --image-tag 0.25.26 --skip-pull --enable-api-proxy \ - -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || echo node)"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_driver.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir /tmp/gh-aw/cache-memory/ --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log - env: - COPILOT_AGENT_RUNNER_TYPE: STANDALONE - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} - GH_AW_ASSETS_ALLOWED_EXTS: ".png,.jpg,.jpeg,.svg" - GH_AW_ASSETS_BRANCH: "assets/${{ github.workflow }}" - GH_AW_ASSETS_MAX_SIZE_KB: 10240 - GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json - GH_AW_PHASE: agent - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_VERSION: dev - GITHUB_API_URL: ${{ github.api_url }} - GITHUB_AW: true - GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows - GITHUB_HEAD_REF: ${{ github.head_ref }} - GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md - GITHUB_WORKSPACE: ${{ github.workspace }} - GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com - GIT_AUTHOR_NAME: github-actions[bot] - GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com - GIT_COMMITTER_NAME: github-actions[bot] - XDG_CONFIG_HOME: /home/runner - - name: Detect Copilot errors - id: detect-copilot-errors - if: always() - continue-on-error: true - run: node "${RUNNER_TEMP}/gh-aw/actions/detect_copilot_errors.cjs" - - name: Configure Git credentials - env: - REPO_NAME: ${{ github.repository }} - SERVER_URL: ${{ github.server_url }} - GITHUB_TOKEN: ${{ github.token }} - run: | - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - git config --global am.keepcr true - # Re-authenticate git with GitHub token - SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" - echo "Git configured with standard GitHub Actions identity" - - name: Copy Copilot session state files to logs - if: always() - continue-on-error: true - run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh" - - name: Stop MCP Gateway - if: always() - continue-on-error: true - env: - MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} - MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} - GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} - run: | - bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" - - name: Redact secrets in logs - if: always() - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); - await main(); - env: - GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' - SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} - SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} - SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Append agent step summary - if: always() - run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh" - - name: Copy Safe Outputs - if: always() - env: - GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - run: | - mkdir -p /tmp/gh-aw - cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true - - name: Ingest agent output - id: collect_output - if: always() - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_API_URL: ${{ github.api_url }} - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs'); - await main(); - - name: Parse agent logs for step summary - if: always() - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); - await main(); - - name: Parse MCP Gateway logs for step summary - if: always() - id: parse-mcp-gateway - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); - await main(); - - name: Print firewall logs - if: always() - continue-on-error: true - env: - AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs - run: | - # Fix permissions on firewall logs so they can be uploaded as artifacts - # AWF runs with sudo, creating files owned by root - sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true - # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) - if command -v awf &> /dev/null; then - awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" - else - echo 'AWF binary not installed, skipping firewall log summary' - fi - - name: Parse token usage for step summary - if: always() - continue-on-error: true - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); - await main(); - - name: Write agent output placeholder if missing - if: always() - run: | - if [ ! -f /tmp/gh-aw/agent_output.json ]; then - echo '{"items":[]}' > /tmp/gh-aw/agent_output.json - fi - - name: Commit cache-memory changes - if: always() - env: - GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory - run: bash "${RUNNER_TEMP}/gh-aw/actions/commit_cache_memory_git.sh" - - name: Upload cache-memory data as artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: always() - with: - name: cache-memory - path: /tmp/gh-aw/cache-memory - # Upload safe-outputs assets for upload_assets job - - name: Upload Safe Outputs Assets - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: safe-outputs-assets - path: /tmp/gh-aw/safeoutputs/assets/ - retention-days: 1 - if-no-files-found: ignore - # Upload safe-outputs upload-artifact staging for the upload_artifact job - - name: Upload upload-artifact staging - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: safe-outputs-upload-artifacts - path: ${{ runner.temp }}/gh-aw/safeoutputs/upload-artifacts/ - retention-days: 1 - if-no-files-found: ignore - - name: Upload agent artifacts - if: always() - continue-on-error: true - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: agent - path: | - /tmp/gh-aw/aw-prompts/prompt.txt - /tmp/gh-aw/sandbox/agent/logs/ - /tmp/gh-aw/redacted-urls.log - /tmp/gh-aw/mcp-logs/ - /tmp/gh-aw/agent_usage.json - /tmp/gh-aw/agent-stdio.log - /tmp/gh-aw/agent/ - /tmp/gh-aw/github_rate_limits.jsonl - /tmp/gh-aw/safeoutputs.jsonl - /tmp/gh-aw/agent_output.json - /tmp/gh-aw/aw-*.patch - /tmp/gh-aw/aw-*.bundle - /tmp/gh-aw/sandbox/firewall/logs/ - /tmp/gh-aw/sandbox/firewall/audit/ - if-no-files-found: ignore - - conclusion: - needs: - - activation - - agent - - detection - - safe_outputs - - update_cache_memory - - upload_assets - if: > - always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || - needs.activation.outputs.stale_lock_file_failed == 'true') - runs-on: ubuntu-slim - permissions: - contents: read - discussions: write - issues: write - concurrency: - group: "gh-aw-conclusion-portfolio-analyst" - cancel-in-progress: false - outputs: - incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} - noop_message: ${{ steps.noop.outputs.noop_message }} - tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} - total_count: ${{ steps.missing_tool.outputs.total_count }} - steps: - - name: Checkout actions folder - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - repository: github/gh-aw - sparse-checkout: | - actions - persist-credentials: false - - name: Setup Scripts - id: setup - uses: ./actions/setup - with: - destination: ${{ runner.temp }}/gh-aw/actions - job-name: ${{ github.job }} - trace-id: ${{ needs.activation.outputs.setup-trace-id }} - - name: Download agent output artifact - id: download-agent-output - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: agent - path: /tmp/gh-aw/ - - name: Setup agent output environment variable - id: setup-agent-output-env - if: steps.download-agent-output.outcome == 'success' - run: | - mkdir -p /tmp/gh-aw/ - find "/tmp/gh-aw/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" - - name: Process no-op messages - id: noop - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_NOOP_MAX: "1" - GH_AW_WORKFLOW_NAME: "Automated Portfolio Analyst" - GH_AW_TRACKER_ID: "portfolio-analyst-weekly" - GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_NOOP_REPORT_AS_ISSUE: "true" - with: - github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs'); - await main(); - - name: Log detection run - id: detection_runs - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Automated Portfolio Analyst" - GH_AW_TRACKER_ID: "portfolio-analyst-weekly" - GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} - GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} - with: - github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs'); - await main(); - - name: Record missing tool - id: missing_tool - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" - GH_AW_WORKFLOW_NAME: "Automated Portfolio Analyst" - GH_AW_TRACKER_ID: "portfolio-analyst-weekly" - with: - github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs'); - await main(); - - name: Record incomplete - id: report_incomplete - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" - GH_AW_WORKFLOW_NAME: "Automated Portfolio Analyst" - GH_AW_TRACKER_ID: "portfolio-analyst-weekly" - with: - github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs'); - await main(); - - name: Handle agent failure - id: handle_agent_failure - if: always() - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Automated Portfolio Analyst" - GH_AW_TRACKER_ID: "portfolio-analyst-weekly" - GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "portfolio-analyst" - GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "12" - GH_AW_ENGINE_ID: "copilot" - GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} - GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} - GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} - GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} - GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} - GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} - GH_AW_CREATE_DISCUSSION_ERRORS: ${{ needs.safe_outputs.outputs.create_discussion_errors }} - GH_AW_CREATE_DISCUSSION_ERROR_COUNT: ${{ needs.safe_outputs.outputs.create_discussion_error_count }} - GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} - GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} - GH_AW_GROUP_REPORTS: "false" - GH_AW_FAILURE_REPORT_AS_ISSUE: "true" - GH_AW_TIMEOUT_MINUTES: "20" - with: - github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs'); - await main(); - - detection: - needs: - - activation - - agent - if: > - always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true') - runs-on: ubuntu-latest - permissions: - contents: read - outputs: - detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} - detection_reason: ${{ steps.detection_conclusion.outputs.reason }} - detection_success: ${{ steps.detection_conclusion.outputs.success }} - steps: - - name: Checkout actions folder - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - repository: github/gh-aw - sparse-checkout: | - actions - persist-credentials: false - - name: Setup Scripts - id: setup - uses: ./actions/setup - with: - destination: ${{ runner.temp }}/gh-aw/actions - job-name: ${{ github.job }} - trace-id: ${{ needs.activation.outputs.setup-trace-id }} - - name: Download agent output artifact - id: download-agent-output - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: agent - path: /tmp/gh-aw/ - - name: Setup agent output environment variable - id: setup-agent-output-env - if: steps.download-agent-output.outcome == 'success' - run: | - mkdir -p /tmp/gh-aw/ - find "/tmp/gh-aw/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" - - name: Checkout repository for patch context - if: needs.agent.outputs.has_patch == 'true' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - # --- Threat Detection --- - - name: Clean stale firewall files from agent artifact - run: | - rm -rf /tmp/gh-aw/sandbox/firewall/logs - rm -rf /tmp/gh-aw/sandbox/firewall/audit - - name: Download container images - run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.26 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.26 ghcr.io/github/gh-aw-firewall/squid:0.25.26 - - name: Check if detection needed - id: detection_guard - if: always() - env: - OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} - HAS_PATCH: ${{ needs.agent.outputs.has_patch }} - run: | - if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then - echo "run_detection=true" >> "$GITHUB_OUTPUT" - echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" - else - echo "run_detection=false" >> "$GITHUB_OUTPUT" - echo "Detection skipped: no agent outputs or patches to analyze" - fi - - name: Clear MCP configuration for detection - if: always() && steps.detection_guard.outputs.run_detection == 'true' - run: | - rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" - rm -f /home/runner/.copilot/mcp-config.json - rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" - - name: Prepare threat detection files - if: always() && steps.detection_guard.outputs.run_detection == 'true' - run: | - mkdir -p /tmp/gh-aw/threat-detection/aw-prompts - cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true - cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true - for f in /tmp/gh-aw/aw-*.patch; do - [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true - done - for f in /tmp/gh-aw/aw-*.bundle; do - [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true - done - echo "Prepared threat detection files:" - ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true - - name: Setup threat detection - if: always() && steps.detection_guard.outputs.run_detection == 'true' - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - WORKFLOW_NAME: "Automated Portfolio Analyst" - WORKFLOW_DESCRIPTION: "Weekly portfolio analyst that identifies cost reduction opportunities (20%+) while improving workflow reliability" - HAS_PATCH: ${{ needs.agent.outputs.has_patch }} - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs'); - await main(); - - name: Ensure threat-detection directory and log - if: always() && steps.detection_guard.outputs.run_detection == 'true' - run: | - mkdir -p /tmp/gh-aw/threat-detection - touch /tmp/gh-aw/threat-detection/detection.log - - name: Install GitHub Copilot CLI - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.21 - env: - GH_HOST: github.com - - name: Install AWF binary - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.26 - - name: Execute GitHub Copilot CLI - if: always() && steps.detection_guard.outputs.run_detection == 'true' - id: detection_agentic_execution - # Copilot CLI tool arguments (sorted): - timeout-minutes: 20 - run: | - set -o pipefail - touch /tmp/gh-aw/agent-step-summary.md - GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) - export GH_AW_NODE_BIN - (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) - # shellcheck disable=SC1003 - sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,github.com,host.docker.internal,telemetry.enterprise.githubcopilot.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --image-tag 0.25.26 --skip-pull --enable-api-proxy \ - -- /bin/bash -c 'GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || echo node)"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_driver.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log - env: - COPILOT_AGENT_RUNNER_TYPE: STANDALONE - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} - GH_AW_PHASE: detection - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_VERSION: dev - GITHUB_API_URL: ${{ github.api_url }} - GITHUB_AW: true - GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows - GITHUB_HEAD_REF: ${{ github.head_ref }} - GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md - GITHUB_WORKSPACE: ${{ github.workspace }} - GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com - GIT_AUTHOR_NAME: github-actions[bot] - GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com - GIT_COMMITTER_NAME: github-actions[bot] - XDG_CONFIG_HOME: /home/runner - - name: Upload threat detection log - if: always() && steps.detection_guard.outputs.run_detection == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: detection - path: /tmp/gh-aw/threat-detection/detection.log - if-no-files-found: ignore - - name: Parse and conclude threat detection - id: detection_conclusion - if: always() - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} - GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs'); - await main(); - - safe_outputs: - needs: - - activation - - agent - - detection - if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' - runs-on: ubuntu-slim - permissions: - contents: read - discussions: write - issues: write - timeout-minutes: 15 - env: - GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/portfolio-analyst" - GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} - GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} - GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} - GH_AW_ENGINE_ID: "copilot" - GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} - GH_AW_TRACKER_ID: "portfolio-analyst-weekly" - GH_AW_WORKFLOW_ID: "portfolio-analyst" - GH_AW_WORKFLOW_NAME: "Automated Portfolio Analyst" - outputs: - code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} - code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} - create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} - create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} - process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} - process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} - upload_artifact_count: ${{ steps.process_safe_outputs.outputs.upload_artifact_count }} - upload_artifact_slot_0_tmp_id: ${{ steps.process_safe_outputs.outputs.slot_0_tmp_id }} - upload_artifact_slot_1_tmp_id: ${{ steps.process_safe_outputs.outputs.slot_1_tmp_id }} - upload_artifact_slot_2_tmp_id: ${{ steps.process_safe_outputs.outputs.slot_2_tmp_id }} - steps: - - name: Checkout actions folder - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - repository: github/gh-aw - sparse-checkout: | - actions - persist-credentials: false - - name: Setup Scripts - id: setup - uses: ./actions/setup - with: - destination: ${{ runner.temp }}/gh-aw/actions - job-name: ${{ github.job }} - trace-id: ${{ needs.activation.outputs.setup-trace-id }} - safe-output-artifact-client: 'true' - - name: Download agent output artifact - id: download-agent-output - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: agent - path: /tmp/gh-aw/ - - name: Setup agent output environment variable - id: setup-agent-output-env - if: steps.download-agent-output.outcome == 'success' - run: | - mkdir -p /tmp/gh-aw/ - find "/tmp/gh-aw/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" - - name: Configure GH_HOST for enterprise compatibility - id: ghes-host-config - shell: bash - run: | - # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct - # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. - GH_HOST="${GITHUB_SERVER_URL#https://}" - GH_HOST="${GH_HOST#http://}" - echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" - - name: Download upload-artifact staging - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: safe-outputs-upload-artifacts - path: ${{ runner.temp }}/gh-aw/safeoutputs/upload-artifacts/ - - name: Process Safe Outputs - id: process_safe_outputs - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_ALLOWED_DOMAINS: "*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" - GITHUB_SERVER_URL: ${{ github.server_url }} - GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_discussion\":{\"category\":\"audits\",\"close_older_discussions\":true,\"expires\":24,\"fallback_to_issue\":true,\"max\":1,\"title_prefix\":\"[portfolio] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{},\"upload_artifact\":{\"max-size-bytes\":104857600,\"max-uploads\":3,\"retention-days\":30,\"skip-archive\":true},\"upload_asset\":{\"allowed-exts\":[\".png\",\".jpg\",\".jpeg\",\".svg\"],\"branch\":\"assets/${{ github.workflow }}\",\"max\":5,\"max-size\":10240}}" - with: - github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs'); - await main(); - - name: Upload Safe Outputs Items - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: safe-outputs-items - path: | - /tmp/gh-aw/safe-output-items.jsonl - /tmp/gh-aw/temporary-id-map.json - if-no-files-found: ignore - - update_cache_memory: - needs: - - activation - - agent - - detection - if: > - always() && (needs.detection.result == 'success' || needs.detection.result == 'skipped') && - needs.agent.result == 'success' - runs-on: ubuntu-slim - permissions: - contents: read - env: - GH_AW_WORKFLOW_ID_SANITIZED: portfolioanalyst - steps: - - name: Checkout actions folder - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - repository: github/gh-aw - sparse-checkout: | - actions - persist-credentials: false - - name: Setup Scripts - id: setup - uses: ./actions/setup - with: - destination: ${{ runner.temp }}/gh-aw/actions - job-name: ${{ github.job }} - trace-id: ${{ needs.activation.outputs.setup-trace-id }} - - name: Download cache-memory artifact (default) - id: download_cache_default - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - continue-on-error: true - with: - name: cache-memory - path: /tmp/gh-aw/cache-memory - - name: Check if cache-memory folder has content (default) - id: check_cache_default - shell: bash - run: | - if [ -d "/tmp/gh-aw/cache-memory" ] && [ "$(ls -A /tmp/gh-aw/cache-memory 2>/dev/null)" ]; then - echo "has_content=true" >> "$GITHUB_OUTPUT" - else - echo "has_content=false" >> "$GITHUB_OUTPUT" - fi - - name: Save cache-memory to cache (default) - if: steps.check_cache_default.outputs.has_content == 'true' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - key: memory-none-nopolicy-trending-data-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}-${{ github.run_id }} - path: /tmp/gh-aw/cache-memory - - upload_assets: - needs: - - activation - - agent - if: (!cancelled()) && needs.agent.result != 'skipped' && contains(needs.agent.outputs.output_types, 'upload_asset') - runs-on: ubuntu-slim - permissions: - contents: write - timeout-minutes: 10 - outputs: - branch_name: ${{ steps.upload_assets.outputs.branch_name }} - published_count: ${{ steps.upload_assets.outputs.published_count }} - steps: - - name: Checkout actions folder - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - repository: github/gh-aw - sparse-checkout: | - actions - persist-credentials: false - - name: Setup Scripts - id: setup - uses: ./actions/setup - with: - destination: ${{ runner.temp }}/gh-aw/actions - job-name: ${{ github.job }} - trace-id: ${{ needs.activation.outputs.setup-trace-id }} - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - fetch-depth: 0 - - name: Configure Git credentials - env: - REPO_NAME: ${{ github.repository }} - SERVER_URL: ${{ github.server_url }} - GITHUB_TOKEN: ${{ github.token }} - run: | - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - git config --global am.keepcr true - # Re-authenticate git with GitHub token - SERVER_URL_STRIPPED="${SERVER_URL#https://}" - git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" - echo "Git configured with standard GitHub Actions identity" - - name: Download assets - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: safe-outputs-assets - path: /tmp/gh-aw/safeoutputs/assets/ - - name: List downloaded asset files - continue-on-error: true - run: | - echo "Downloaded asset files:" - find /tmp/gh-aw/safeoutputs/assets/ -maxdepth 1 -ls - - name: Download agent output artifact - id: download-agent-output - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: agent - path: /tmp/gh-aw/ - - name: Setup agent output environment variable - id: setup-agent-output-env - if: steps.download-agent-output.outcome == 'success' - run: | - mkdir -p /tmp/gh-aw/ - find "/tmp/gh-aw/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" - - name: Push assets - id: upload_assets - uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 - env: - GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_ASSETS_BRANCH: "assets/${{ github.workflow }}" - GH_AW_ASSETS_MAX_SIZE_KB: 10240 - GH_AW_ASSETS_ALLOWED_EXTS: ".png,.jpg,.jpeg,.svg" - GH_AW_WORKFLOW_NAME: "Automated Portfolio Analyst" - GH_AW_TRACKER_ID: "portfolio-analyst-weekly" - GH_AW_ENGINE_ID: "copilot" - GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} - with: - github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/upload_assets.cjs'); - await main(); - - name: Restore actions folder - if: always() - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - repository: github/gh-aw - sparse-checkout: | - actions/setup - sparse-checkout-cone-mode: true - persist-credentials: false - diff --git a/.github/workflows/portfolio-analyst.md b/.github/workflows/portfolio-analyst.md deleted file mode 100644 index 6a3f7766f95..00000000000 --- a/.github/workflows/portfolio-analyst.md +++ /dev/null @@ -1,594 +0,0 @@ ---- -description: Weekly portfolio analyst that identifies cost reduction opportunities (20%+) while improving workflow reliability -on: - schedule: weekly on monday around 09:00 - workflow_dispatch: -permissions: - contents: read - actions: read - issues: read - pull-requests: read -tracker-id: portfolio-analyst-weekly -engine: copilot -network: - allowed: [python] -tools: - mount-as-clis: true - agentic-workflows: - github: - toolsets: [default] - bash: ["*"] -steps: - - name: Install gh-aw CLI - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - if gh extension list | grep -q "github/gh-aw"; then - gh extension upgrade gh-aw || true - else - gh extension install github/gh-aw - fi - gh aw --version - - name: Download logs from last 30 days - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - mkdir -p /tmp/portfolio-logs - gh aw logs --start-date -30d -c 5000 -o /tmp/portfolio-logs --json > /tmp/portfolio-logs/summary.json -safe-outputs: - upload-artifact: - max-uploads: 3 - retention-days: 30 - skip-archive: true -timeout-minutes: 20 -imports: - - uses: shared/daily-audit-discussion.md - with: - title-prefix: "[portfolio] " - expires: 1d - - shared/reporting.md - - shared/jqschema.md - - shared/trending-charts-simple.md -features: - mcp-cli: true ---- - -# Automated Portfolio Analyst - -You are an expert workflow portfolio analyst focused on identifying cost reduction opportunities while improving reliability. - -## ⚠️ Critical: Pre-Downloaded Data Location - -**All workflow execution data has been pre-downloaded for you in the previous workflow step.** - -- **JSON Summary**: `/tmp/portfolio-logs/summary.json` - Contains all metrics and run data you need -- **Run Logs**: `/tmp/portfolio-logs/run-{database-id}/` - Individual run logs (if needed for detailed analysis) - -**DO NOT call `gh aw logs` or any GitHub CLI commands** - they will not work in your environment. All data you need is in the summary.json file. - -## Mission - -Analyze all agentic workflows in this repository weekly to identify opportunities to reduce costs while maintaining or improving reliability. Complete the entire analysis in under 60 seconds by focusing on high-impact issues. - -**Important**: Always generate a report, even with limited data. Be transparent about data limitations and adjust recommendations accordingly. - -## Current Context - -- **Repository**: ${{ github.repository }} -- **Analysis Date**: Use `date +%Y-%m-%d` command to get current date -- **Target**: Identify all cost reduction opportunities (aim for 20%+ when data permits) -- **Time Budget**: 60 seconds - -## Visualization Requirements - -**Generate visual charts to create a dashboard-style report**. The report should be concise, scannable, and use charts instead of long text descriptions. - -### Required Charts - -Create these charts using Python with matplotlib/seaborn and save to `/tmp/gh-aw/python/charts/`: - -1. **Cost Trends Chart** (`cost_trends.png`) - - Line chart showing daily workflow costs over the last 30 days - - Highlight the overall trend (increasing/decreasing) - - Include 7-day moving average - -2. **Top Spenders Chart** (`top_spenders.png`) - - Horizontal bar chart of top 10 workflows by total cost - - Show actual dollar amounts - - Color code by health status (green=healthy, yellow=warning, red=critical) - -3. **Failure Rates Chart** (`failure_rates.png`) - - Bar chart showing workflows with >30% failure rate - - Display failure percentage and wasted cost - - Sort by wasted cost (highest first) - -4. **Success Rate Overview** (`success_overview.png`) - - Pie or donut chart showing overall success/failure/cancelled distribution - - Include percentages and counts - -### Chart Requirements - -- **High quality**: 300 DPI, 12x7 inch figures -- **Clear labels**: Title, axis labels, legends -- **Professional styling**: Use seaborn whitegrid style -- **Consistent colors**: Use a professional color palette -- **Upload all charts** using the `upload asset` tool to get URLs -- **Embed in report**: Include charts in the discussion using markdown image syntax - -### Data Preparation - -Extract data from `/tmp/portfolio-logs/summary.json` and prepare it as CSV files in `/tmp/gh-aw/python/data/` before generating charts. Example: - -```python -import pandas as pd -import json - -# Load summary data -with open('/tmp/portfolio-logs/summary.json', 'r') as f: - data = json.load(f) - -# Prepare daily cost data -runs_df = pd.DataFrame(data['runs']) -runs_df['date'] = pd.to_datetime(runs_df['created_at']).dt.date -daily_costs = runs_df.groupby('date')['estimated_cost'].sum() -daily_costs.to_csv('/tmp/gh-aw/python/data/daily_costs.csv') -``` - -## Analysis Framework - -### Phase 0: Important Setup Notes - -**DO NOT CALL `gh aw logs` OR ANY `gh` COMMANDS** - These commands will not work in your environment and will fail. - -The workflow logs have already been downloaded for you in the previous step. The data is available at: -- **JSON Summary File**: `/tmp/portfolio-logs/summary.json` (contains all metrics and run data) -- **Individual Run Logs Directory**: `/tmp/portfolio-logs/run-{database-id}/` (detailed logs for each workflow run) - -All the data you need has been pre-downloaded. Read from these files instead of calling `gh` commands. - -### Phase 1: Data Collection (10 seconds) - -Collect execution data from the pre-downloaded logs: - -```bash -# Read the pre-downloaded JSON summary (this file contains ALL the data you need) -cat /tmp/portfolio-logs/summary.json | jq '.' - -# The summary.json file contains: -# - .summary: Aggregate metrics (total runs, tokens, cost, errors, warnings) -# - .runs: Array of all workflow runs with detailed metrics per run -# - .logs_location: Base directory where run logs are stored - -# Get total number of runs analyzed -cat /tmp/portfolio-logs/summary.json | jq '.summary.total_runs' - -# Get all runs with their metrics -cat /tmp/portfolio-logs/summary.json | jq '.runs[]' - -# Get list of all agentic workflows in the repository -find .github/workflows/ -name '*.md' -type f - -# Individual run logs are stored in subdirectories (if you need detailed logs) -find /tmp/portfolio-logs -type d -name "run-*" -``` - -**Key Metrics to Extract (from summary.json .runs array):** -- `database_id` - Unique run identifier -- `workflow_name` - Name of the workflow -- `estimated_cost` - **Real cost per run calculated from actual token usage** (field name says "estimated" but contains calculated cost from actual usage) -- `token_usage` - Actual token consumption -- `duration` - Actual runtime (formatted as string like "5m30s") -- `conclusion` - Success/failure status (success, failure, cancelled) -- `created_at` - When the run was executed (ISO 8601 timestamp) -- `error_count` - Number of errors in the run -- `warning_count` - Number of warnings in the run - -**Calculate from real data:** -- Total runs in last 30 days: Use `.summary.total_runs` or count `.runs` array -- Success/failure counts: Count runs where `.conclusion` == "success" or "failure" -- Last run date: Find latest `.created_at` timestamp -- Monthly cost: Use `.summary.total_cost` (sum of all runs' estimated_cost) -- Average cost per run: `.summary.total_cost / .summary.total_runs` - -**Triage Early:** -- Skip workflows with 100% success rate, normal frequency, and last run < 7 days -- Focus 80% of analysis time on top 20% of issues - -**Handling Limited Data:** -- If limited data (< 10 workflow runs), acknowledge this upfront in the report -- Provide what insights are possible based on available data -- Be transparent about limitations and caveats -- Still generate a report - don't refuse due to insufficient data - -### Phase 2: Five-Dimension Analysis (15 seconds) - -Analyze each workflow across five dimensions: - -#### 1. Overlap Risk -- Identify workflows with similar triggers -- Detect duplicate functionality -- Find workflows that could be consolidated - -#### 2. Business Value -- Check last run date (flag if >60 days) -- Review trigger patterns (flag if never triggered) -- Assess actual usage vs. configured schedule - -#### 3. Cost Efficiency -- Use **ACTUAL cost data** from downloaded JSON files -- Sum `estimated_cost` from all runs in the last 30 days for real monthly cost -- **Flag workflows costing >$10/month** (based on actual spend, not estimates) -- Identify over-scheduled workflows (daily when weekly would suffice) - -#### 4. Operational Health -- Calculate failure rate -- **Flag workflows with >30% failure rate** -- Identify patterns in failures - -#### 5. Security Posture -- Review permissions (flag excessive permissions) -- Check network allowlists -- Assess safe-output usage - -### Phase 3: Triage Categories (5 seconds) - -Sort workflows into three categories: - -**Healthy (Skip):** -- <30% failure rate -- Last run <60 days -- Cost <$10/month -- No obvious duplicates -- ~60-70% of workflows should be in this category - -**Removal Candidates:** -- No runs in 60+ days -- Zero triggers in last 30 days -- Replaced by other workflows - -**Problematic (Requires Analysis):** -- >30% failure rate -- Cost >$10/month -- Clear duplicates -- Over-scheduled (daily when weekly suffices) - -### Phase 4: High-Impact Focus (20 seconds) - -Focus exclusively on: - -1. **Workflows costing >$10/month** - Analyze for frequency reduction -2. **Workflows with >30% failure rate** - Calculate wasted spending -3. **Clear duplicates** - Calculate consolidation savings -4. **Over-scheduled workflows** - Calculate frequency reduction savings - -Skip everything else to stay within time budget. - -### Phase 5: Savings Calculation (10 seconds) - -Calculate specific dollar amounts using **ACTUAL cost data from downloaded files**: - -#### Strategy 1: Remove Unused Workflows -```bash -# Read cost data from the JSON summary for specific workflows -cat /tmp/portfolio-logs/summary.json | jq '.runs[] | select(.workflow_name == "workflow-name") | .estimated_cost' | jq -s 'add' - -For each workflow with no runs in 60+ days: -- Current monthly cost: Sum of estimated_cost from last 30 days -- Savings: $X/month (actual spend, not estimate) -- Total savings: Sum all -``` - -#### Strategy 2: Reduce Schedule Frequency -```bash -# Get actual runs and cost from the JSON summary -cat /tmp/portfolio-logs/summary.json | jq '[.runs[] | select(.workflow_name == "workflow-name")] | {runs: length, cost: map(.estimated_cost) | add}' - -For each over-scheduled workflow: -- Current frequency: Count runs in last 30 days from summary.json -- Average cost per run: total_cost / total_runs (from actual data) -- Recommended: Weekly (4 runs/month) -- Savings: (current_runs - 4) × avg_cost_per_run = $Y/month -``` - -#### Strategy 3: Consolidate Duplicates -```bash -# Get cost for each duplicate workflow from the JSON summary -cat /tmp/portfolio-logs/summary.json | jq '[.runs[] | select(.workflow_name == "workflow-1")] | map(.estimated_cost) | add' -cat /tmp/portfolio-logs/summary.json | jq '[.runs[] | select(.workflow_name == "workflow-2")] | map(.estimated_cost) | add' - -For each duplicate set: -- Number of duplicates: N -- Cost per workflow: $X (from summary.json actual data) -- Savings: (N-1) × $X/month -``` - -#### Strategy 4: Fix High-Failure Workflows -```bash -# Get failure rate and cost from the JSON summary -cat /tmp/portfolio-logs/summary.json | jq '[.runs[] | select(.workflow_name == "workflow-name" and .conclusion == "failure")] | map(.estimated_cost) | add' - -For each workflow with >30% failure rate: -- Total runs: Count from summary.json -- Failed runs: Count where conclusion == "failure" -- Failure rate: (failed_runs / total_runs) × 100 -- Wasted spending: Sum of estimated_cost for failed runs -- Potential savings: $Y/month (actual wasted cost on failures) -``` - -**Total Savings Target: Aim for ≥20% of current spending (adjust expectations for limited data)** - -## Output Requirements - -Generate a **concise, visual dashboard-style report** under 1500 words with embedded charts. - -### Report Structure - -**CRITICAL**: The report must be visual and scannable. Generate all required charts FIRST, upload them as assets, then create the discussion with embedded charts. - -```markdown -# 📊 Portfolio Dashboard - [DATE] - -## Quick Overview - -[2-3 sentences summarizing key findings, total costs, and potential savings] - -## Visual Summary - -### Cost Trends (Last 30 Days) - -![Cost Trends](URL_FROM_UPLOAD_ASSET_FOR_cost_trends.png) - -**Key Insights**: -- Daily average: $[X] -- Trend: [Increasing/Decreasing/Stable] ([Y]% change) -- Monthly total: $[Z] - -### Top Cost Drivers - -![Top Spenders](URL_FROM_UPLOAD_ASSET_FOR_top_spenders.png) - -Top 3 workflows account for [X]% of total cost: -1. `workflow-1.md` - $[X]/month ([status]) -2. `workflow-2.md` - $[Y]/month ([status]) -3. `workflow-3.md` - $[Z]/month ([status]) - -### Failure Analysis - -![Failure Rates](URL_FROM_UPLOAD_ASSET_FOR_failure_rates.png) - -**Wasted Spend**: $[X]/month on failed runs -- [N] workflows with >30% failure rate -- [M] workflows with 100% failure rate (should be disabled) - -### Overall Health - -![Success Overview](URL_FROM_UPLOAD_ASSET_FOR_success_overview.png) - -- ✅ Success: [X]% ([N] runs) -- ❌ Failure: [Y]% ([M] runs) -- ⏸️ Cancelled: [Z]% ([P] runs) - -## 💰 Cost Reduction Opportunities - -**Total Potential Savings: $[X]/month ([Y]% reduction)** - -
-Strategy 1: Fix High-Failure Workflows - $[X]/month - -List workflows with >30% failure rate, showing: -- Workflow name and file -- Failure rate percentage -- Wasted cost per month -- Recommended fix (1-2 lines) - -
- -
-Strategy 2: Reduce Over-Scheduling - $[Y]/month - -List over-scheduled workflows with: -- Current frequency (runs/month) -- Recommended frequency -- Savings calculation - -
- -
-Strategy 3: Disable Failed Workflows - $[Z]/month - -List workflows with 100% failure rate or no successful runs. - -
- -
-Strategy 4: Remove Unused Workflows - $[W]/month - -List workflows with no runs in 60+ days. - -
- -## 🎯 Priority Actions - -1. **CRITICAL** - [Highest impact action with specific workflow and cost] -2. **HIGH** - [Second highest impact action] -3. **MEDIUM** - [Third priority action] - -## 📈 Data Quality - -- **Period Analyzed**: [Actual dates covered] -- **Total Runs**: [N] workflow runs -- **Workflows**: [M] total, [X] executed, [Y] not run -- **Confidence**: [High/Medium/Low] based on [reasoning] - ---- - -**Methodology**: Analysis based on actual workflow execution data from `gh aw logs` for the last 30 days. Costs calculated from real token usage, not estimates. -``` - -### Key Requirements - -1. **Generate Charts First** - - Create all 4 required charts using Python - - Save to `/tmp/gh-aw/python/charts/` - - Upload each using `upload asset` tool - - Get URLs for embedding - -2. **Visual Focus** - - Charts tell the story, not long text - - Use bullet points and short paragraphs - - Expand details in collapsible sections - - Keep overview section scannable - -3. **Dashboard Layout** - - Visual Summary section with all charts upfront - - Brief insights under each chart (2-4 bullet points) - - Detailed recommendations in collapsible details sections - - Priority actions as numbered list - -4. **Conciseness** - - Target: 1000-1500 words total - - Each strategy section: <200 words - - Use tables for comparing workflows - - Focus on actionable items only - -5. **Consistency** - - Same chart types every week - - Same section structure - - Same visual styling (colors, fonts) - - Easy to compare week-over-week - -## Critical Guidelines - -### Handling Limited Data Scenarios - -**ALWAYS generate a report**, regardless of data availability. Never refuse or fail due to insufficient data. - -When data is limited (examples: only today's runs, < 10 total runs, < 7 days of history): -1. **Acknowledge limitations upfront** in the "Data Availability" section -2. **Document the actual period covered** (e.g., "Last 24 hours" vs "Last 30 days") -3. **State confidence level** (Low/Medium/High based on data volume) -4. **Provide caveats**: Explain that patterns may not be representative -5. **Make conservative recommendations**: Focus on obvious issues (100% failure rates, never-run workflows) -6. **Avoid extrapolation**: Don't project limited data to full month without caveats -7. **Still deliver value**: Even limited data can identify clear problems - -Example minimal data report format: -```markdown -## Data Availability - -⚠️ **Limited Data Warning**: Only 8 workflow runs available from the last 24 hours. -- **Confidence Level**: Low - Single day snapshot only -- **Recommendations**: Conservative - focusing on obvious issues only -- **Next Steps**: Re-run analysis after accumulating 7+ days of data -``` - -### Use Real Data, Not Guesswork -- **DO NOT call `gh aw logs` or any `gh` commands** - they will not work in your environment -- **Read from the pre-downloaded JSON file `/tmp/portfolio-logs/summary.json`** - all workflow data is in this single file -- **Use calculated costs** - the `estimated_cost` field in each run contains costs calculated from actual token usage -- **Parse JSON with jq** - extract precise metrics from the summary.json file -- **Sum actual costs** - add up `estimated_cost` for all runs in the `.runs` array -- **Calculate from actuals** - failure rates, run frequency, cost per run all from real workflow execution data in summary.json - -### Speed Optimization -- **Skip healthy workflows** - Don't waste time analyzing what works -- **Focus on high-impact only** - Workflows >$10/month or >30% failure (from actual data) -- **Read from summary.json** - All data is in a single pre-downloaded JSON file at `/tmp/portfolio-logs/summary.json` -- **Use templates** - Pre-format output structure - -### Precision Requirements -- **Exact filenames** - Include `.md` extension -- **Exact line numbers** - Specify which lines to modify -- **Copy-paste snippets** - Show before/after for each fix -- **Dollar amounts** - Use actual costs from downloaded logs, not estimates or ranges -- **Show calculations** - Display how you calculated savings from actual data - -### Quality Standards -- **<1500 words** - Be very concise, let charts tell the story -- **Visual first** - Generate all 4 charts before writing report -- **Dashboard style** - Scannable, consistent format week-over-week -- **<1 hour per fix** - Only recommend simple changes -- **Copy-paste ready** - Every fix should be implementable via copy-paste -- **Verify math** - Ensure savings calculations are accurate - -### Visualization Workflow - -**CRITICAL ORDER OF OPERATIONS**: - -1. **Data Preparation** (5 seconds) - - Extract data from summary.json - - Create CSV files in `/tmp/gh-aw/python/data/` - -2. **Generate Charts** (15 seconds) - - Create all 4 required charts using Python - - Save to `/tmp/gh-aw/python/charts/` - - Verify files exist before uploading - -3. **Upload Assets** (10 seconds) - - Upload each chart using `upload asset` tool - - Save the returned URLs - -4. **Create Report** (20 seconds) - - Use the dashboard template - - Embed charts using markdown image syntax - - Keep text concise, let visuals speak - - Use collapsible details sections for lengthy content - -**Example Python Script Structure**: -```python -#!/usr/bin/env python3 -import pandas as pd -import matplotlib.pyplot as plt -import seaborn as sns -import json - -# Load data -with open('/tmp/portfolio-logs/summary.json', 'r') as f: - data = json.load(f) - -# Prepare dataframes -runs_df = pd.DataFrame(data['runs']) -runs_df['date'] = pd.to_datetime(runs_df['created_at']).dt.date - -# Set style once -sns.set_style("whitegrid") -sns.set_palette("husl") - -# Generate all 4 charts -# 1. Cost trends -# 2. Top spenders -# 3. Failure rates -# 4. Success overview - -print("✅ All charts generated") -``` - -### Triage Rules -- **60-70% should be skipped** - Most workflows should be healthy (when sufficient data available) -- **Focus 80% of content on 20% of issues** - High-impact problems only -- **Clear categories** - Remove, Reduce, Consolidate, or Fix -- **Evidence-based** - Use actual run data from downloaded files, not assumptions or estimates -- **Never refuse analysis** - Generate a report even with 1 day of data; just document the limitations - -## Success Criteria - -✅ Analysis completes in <60 seconds -✅ **All 4 required charts generated** (cost trends, top spenders, failure rates, success overview) -✅ **Charts uploaded as assets** and embedded in report -✅ Uses **real data from the pre-downloaded summary.json file**, not estimates -✅ **Always generates a report**, even with limited data -✅ **Dashboard-style format** - visual, scannable, consistent structure -✅ Identifies cost savings opportunities based on available data (aim for ≥20% when data permits) -✅ Clearly documents data limitations and confidence level -✅ Report is <1500 words with majority of insights conveyed through charts -✅ Detailed recommendations in collapsible `
` sections -✅ Every recommendation includes exact line numbers -✅ Every recommendation includes before/after snippets -✅ Every fix takes <1 hour to implement -✅ Math adds up correctly (all costs from actual data in summary.json) -✅ Healthy workflows are briefly mentioned but not analyzed -✅ All dollar amounts are from actual workflow execution data - -Begin your analysis now. **FIRST**: Generate all 4 required charts from `/tmp/portfolio-logs/summary.json` and upload them as assets. **THEN**: Create the dashboard-style discussion with embedded chart URLs. Read from the pre-downloaded JSON file at `/tmp/portfolio-logs/summary.json` to get real execution data for all workflows. This file contains everything you need: summary metrics and individual run data. DO NOT attempt to call `gh aw logs` or any `gh` commands - they will not work. Move fast, focus on high-impact issues, deliver actionable recommendations based on actual costs, and make the report visual and scannable. - -{{#import shared/noop-reminder.md}} diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index e4acbc20730..4018e4238ba 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -283,6 +283,7 @@ export default defineConfig({ { label: 'LabelOps', link: '/patterns/label-ops/' }, { label: 'MultiRepoOps', link: '/patterns/multi-repo-ops/' }, { label: 'Monitoring', link: '/patterns/monitoring/' }, + { label: 'Agentic Observability Kit', link: '/patterns/agentic-observability-kit/' }, { label: 'Orchestration', link: '/patterns/orchestration/' }, { label: 'ProjectOps', link: '/patterns/project-ops/' }, { label: 'ResearchPlanAssignOps', link: '/patterns/research-plan-assign-ops/' }, diff --git a/docs/src/content/docs/agent-factory-status.mdx b/docs/src/content/docs/agent-factory-status.mdx index 8dc07cd272d..16086b0063c 100644 --- a/docs/src/content/docs/agent-factory-status.mdx +++ b/docs/src/content/docs/agent-factory-status.mdx @@ -25,7 +25,6 @@ These are experimental agentic workflows used by the GitHub Next team to learn, | [Artifacts Summary](https://github.com/github/gh-aw/blob/main/.github/workflows/artifacts-summary.md) | copilot | [![Artifacts Summary](https://github.com/github/gh-aw/actions/workflows/artifacts-summary.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/artifacts-summary.lock.yml) | - | - | | [Auto-Assign Issue](https://github.com/github/gh-aw/blob/main/.github/workflows/daily-assign-issue-to-user.md) | copilot | [![Auto-Assign Issue](https://github.com/github/gh-aw/actions/workflows/daily-assign-issue-to-user.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/daily-assign-issue-to-user.lock.yml) | - | - | | [Auto-Triage Issues](https://github.com/github/gh-aw/blob/main/.github/workflows/auto-triage-issues.md) | copilot | [![Auto-Triage Issues](https://github.com/github/gh-aw/actions/workflows/auto-triage-issues.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/auto-triage-issues.lock.yml) | - | - | -| [Automated Portfolio Analyst](https://github.com/github/gh-aw/blob/main/.github/workflows/portfolio-analyst.md) | copilot | [![Automated Portfolio Analyst](https://github.com/github/gh-aw/actions/workflows/portfolio-analyst.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/portfolio-analyst.lock.yml) | - | - | | [Basic Research Agent](https://github.com/github/gh-aw/blob/main/.github/workflows/research.md) | copilot | [![Basic Research Agent](https://github.com/github/gh-aw/actions/workflows/research.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/research.lock.yml) | - | - | | [Blog Auditor](https://github.com/github/gh-aw/blob/main/.github/workflows/blog-auditor.md) | claude | [![Blog Auditor](https://github.com/github/gh-aw/actions/workflows/blog-auditor.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/blog-auditor.lock.yml) | - | - | | [Bot Detection](https://github.com/github/gh-aw/blob/main/.github/workflows/bot-detection.md) | copilot | [![Bot Detection](https://github.com/github/gh-aw/actions/workflows/bot-detection.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/bot-detection.lock.yml) | `every 6h` | - | @@ -172,6 +171,7 @@ These are experimental agentic workflows used by the GitHub Next team to learn, | [Smoke Crush](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-crush.md) | crush | [![Smoke Crush](https://github.com/github/gh-aw/actions/workflows/smoke-crush.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-crush.lock.yml) | - | - | | [Smoke Gemini](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-gemini.md) | gemini | [![Smoke Gemini](https://github.com/github/gh-aw/actions/workflows/smoke-gemini.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-gemini.lock.yml) | - | - | | [Smoke Multi PR](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-multi-pr.md) | copilot | [![Smoke Multi PR](https://github.com/github/gh-aw/actions/workflows/smoke-multi-pr.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-multi-pr.lock.yml) | - | - | +| [Smoke OpenCode](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-opencode.md) | opencode | [![Smoke OpenCode](https://github.com/github/gh-aw/actions/workflows/smoke-opencode.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-opencode.lock.yml) | - | - | | [Smoke Project](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-project.md) | copilot | [![Smoke Project](https://github.com/github/gh-aw/actions/workflows/smoke-project.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-project.lock.yml) | - | - | | [Smoke Service Ports](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-service-ports.md) | copilot | [![Smoke Service Ports](https://github.com/github/gh-aw/actions/workflows/smoke-service-ports.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-service-ports.lock.yml) | - | - | | [Smoke Temporary ID](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-temporary-id.md) | copilot | [![Smoke Temporary ID](https://github.com/github/gh-aw/actions/workflows/smoke-temporary-id.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-temporary-id.lock.yml) | - | - | diff --git a/docs/src/content/docs/patterns/agentic-observability-kit.md b/docs/src/content/docs/patterns/agentic-observability-kit.md new file mode 100644 index 00000000000..ca4ff955789 --- /dev/null +++ b/docs/src/content/docs/patterns/agentic-observability-kit.md @@ -0,0 +1,363 @@ +--- +title: Agentic Observability Kit +description: Use the built-in Agentic Observability Kit to review agentic workflow behavior, detect regressions, and identify evidence-based repository portfolio cleanup opportunities. +--- + +> [!WARNING] +> **Experimental:** The Agentic Observability Kit is still experimental! Things may break, change, or be removed without deprecation at any time. + +The Agentic Observability Kit reviews recent agentic workflow runs in a repository and produces one operator-facing report. It reads run history, episode rollups, and selective audit details, posts a discussion with the results and opens one escalation issue only when repeated patterns warrant owner action. + +The kit now also includes an evidence-based repository portfolio review. In the same report, maintainers can call out workflows that look stale, overlapping, weakly justified for their recent cost, or consistently overbuilt for the task domain they serve. + +This pattern is useful when a repository has enough agentic activity that per-run inspection is too noisy, but maintainers still need practical answers to questions such as which workflows are drifting, which runs are expensive for their domain, which orchestrated chains are accumulating risk, and which workflows may no longer justify their current form. + +## Scope + +The built-in workflow is repository-scoped. It reads workflow runs for the repository where it is installed and produces one report for that repository. + +At repository scope, the report combines two layers: + +- operational observability for recent runs, episodes, regressions, and control failures +- an evidence-based repository portfolio appendix for overlap, stale workflows, and weakly justified agentic workflows + +The same pattern can be extended to an organization. For org-wide reporting, run the workflow from a central repository or control-plane repository, give it cross-repository read access, and aggregate results across target repositories using the [MultiRepoOps](/gh-aw/patterns/multi-repo-ops/) or [CentralRepoOps](/gh-aw/patterns/central-repo-ops/) patterns. In practice, that means collecting `gh aw logs` or MCP `logs` output per repository, then generating an organization-level rollup. + +At organization or enterprise scope, the same pattern can grow into a broader portfolio-governance rollup. That is where deeper questions such as consolidation, duplicate ownership, shared policy, and fleet-wide prioritization fit best. + +Enterprise-wide use is possible as an architecture, but it is not a single built-in turnkey workflow today. At enterprise scope, the usual design is a control-plane repository that fans out across multiple organizations or repository groups, collects normalized run data, and publishes a portfolio report. This is closer to a fleet-operations pattern than a drop-in single-repository workflow. + +> [!IMPORTANT] +> The built-in Agentic Observability Kit should be treated as the repository-level building block. Organization-wide and enterprise-wide deployments require additional cross-repository authentication, central orchestration, and portfolio aggregation logic. + +## Recommended deployment model + +The recommended approach depends on scope. + +### Single repository + +Use the built-in workflow directly in that repository. This is the intended default and requires the least operational overhead. + +### One organization + +Prefer a central repository or control-plane repository that aggregates results from many repositories. This is usually better than installing an independent copy of the workflow in every repository, because it keeps the reporting logic, authentication, routing, and portfolio rollup in one place. + +Installing the workflow in every repository can still make sense when each repository has its own maintainers and needs a self-contained local report. That model is operationally simpler at first, but it produces fragmented reporting and makes organization-level prioritization harder. + +### Enterprise-wide across multiple organizations + +Prefer the same central aggregation model, but treat it as fleet operations rather than a simple workflow rollout. In practice, that means one or more control-plane repositories collecting normalized data from many repositories or organizations and publishing a portfolio-level report. + +This is the recommended model because enterprise-wide observability usually needs shared policy, shared authentication, shared repository allowlists, and a consistent rollup format. Duplicating the workflow everywhere and trying to reconcile the reports afterward is usually the wrong tradeoff. + +## Deployment by scope + +The practical setup differs by scope. + +### Single repository: install the built-in workflow + +For one repository, use the built-in workflow directly and keep the report local to that repository. + +```aw wrap +--- +on: + schedule: weekly on monday around 08:00 + workflow_dispatch: +permissions: + contents: read + actions: read + issues: read + pull-requests: read + discussions: read +engine: copilot +tools: + agentic-workflows: + github: + toolsets: [default, discussions] +safe-outputs: + create-issue: + title-prefix: "[observability escalation] " + max: 1 +--- + +# Agentic Observability Kit + +Review recent agentic workflow runs in this repository and publish one discussion-oriented report. +``` + +This is the right default when maintainers want repository-local visibility and repository-local ownership, including an evidence-based review of whether current workflows still look justified for the repository. + +### One organization: aggregate from a central repository + +For an organization, prefer one central repository that discovers target repositories, pulls per-repository observability data, and publishes an organization-level rollup. + +```aw wrap +--- +on: + schedule: weekly on monday around 09:00 + workflow_dispatch: +permissions: + contents: read + actions: read + discussions: read +engine: copilot +tools: + github: + github-token: ${{ secrets.GH_AW_READ_ORG_TOKEN }} + toolsets: [repos] + bash: + - "gh aw logs *" +safe-outputs: + create-discussion: + max: 1 +--- + +# Organization Observability Rollup + +Discover target repositories, collect per-repository `gh aw logs --json --repo owner/repo` output, and generate one organization-level summary. +``` + +This is the recommended org-wide model because it centralizes authentication, repository allowlists, aggregation logic, and routing decisions. If each repository needs its own local discussion, install the built-in workflow there too, but treat the central rollup as the broader portfolio view. + +### Enterprise-wide: extend the central aggregation pattern + +For multiple organizations, use one or more control-plane repositories that aggregate across repository groups, business units, or organizations. + +```aw wrap +--- +on: + schedule: weekly on monday around 10:00 + workflow_dispatch: +permissions: + contents: read + actions: read + discussions: read +engine: copilot +tools: + github: + github-token: ${{ secrets.GH_AW_ENTERPRISE_READ_TOKEN }} + toolsets: [repos, orgs] + bash: + - "gh aw logs *" +safe-outputs: + create-discussion: + max: 1 +--- + +# Enterprise Observability Rollup + +Collect normalized observability data across approved organizations and repositories, then publish a portfolio report with shared routing and prioritization. +``` + +This should be treated as fleet operations. The goal is not to replicate the repository-level workflow everywhere and stitch the output together later. The goal is to keep aggregation, policy, and prioritization in one place. + +> [!TIP] +> For org-wide and enterprise-wide deployment, start with a pilot allowlist of repositories before expanding coverage. The central aggregation model is operationally safer when authentication, repo discovery, and rollup logic are still being tuned. + +## What it analyzes + +The kit is built around the deterministic lineage data returned by `gh aw logs`. Instead of treating every workflow run as an isolated event, it prefers `episodes[]` and `edges[]` so orchestrator and worker runs are analyzed as one logical execution. This avoids misreading delegated runs in isolation and makes cost, risk, and control signals easier to attribute. + +The episode rollups include aggregate fields such as total runs, total tokens, total estimated cost, blocked requests, MCP failures, risky nodes, and a suggested routing hint for follow-up. When the summary data is not sufficient, the workflow can audit a small number of individual runs to explain the latest regression, a new MCP failure, or a changed write posture. + +For the portfolio portion of the report, the kit can also use targeted workflow-file inspection to confirm trigger or schedule overlap when recent run data suggests that two workflows may be serving the same job. + +## What it computes + +The kit consumes several classes of signals that are already produced by `gh aw logs` and `gh aw audit`: + +- Episode-level rollups for lineage, risk, blocked requests, MCP failures, and suggested route. +- Per-run metrics such as duration, action minutes, token usage, turns, warnings, and `estimated_cost`. +- Effective Tokens, a normalized token metric that weights input, output, cache-read, and cache-write tokens before applying a per-model multiplier. +- Behavior fingerprint and agentic assessments, which help distinguish overkill workflows from genuinely agentic ones. +- Repository-level portfolio signals derived from repeated overkill assessments, weak recent activity, repeated instability, and possible overlap in workflow purpose, trigger pattern, or schedule. + +Effective Tokens matter because raw token counts alone are not comparable across models or token classes. The implementation normalizes token classes first, then applies a model multiplier. This makes it easier to compare a cache-heavy run against an output-heavy run, or a lightweight model against a more expensive one, without collapsing everything into raw token totals. + +## Visual report form + +The kit can produce a chart-backed report format designed for fast interpretation. Instead of relying only on prose, the discussion can include a `Visual Diagnostics` section with a small number of scientific-style plots that make portfolio and observability signals legible at a glance. + +The visual report is most useful when it is concrete. The kit is designed around four fixed plot types, each answering a different maintainer question. + +### 1. Episode Risk-Cost Frontier + +This plot shows execution episodes in cost-risk space. The x-axis is episode cost, the y-axis is an episode risk score derived from risky nodes, poor-control signals, MCP failures, blocked requests, and escalation eligibility, and the point size reflects run count. + +This is the fastest way to answer: which execution chains sit on the cost-risk frontier and deserve immediate attention? + +### 2. Workflow Stability Matrix + +This plot is a workflow-by-metric heatmap. Each row is a workflow, and the columns represent repeated instability signals such as risky run rate, latest-success fallback rate, resource-heavy assessment rate, poor-control rate, blocked-request incidence, and MCP failure incidence. + +This is the fastest way to answer: which workflows are chronically unstable, and which ones are noisy only in one dimension? + +### 3. Repository Portfolio Map + +This plot is a workflow portfolio scatter plot. The x-axis represents recent cost, the y-axis represents an evidence-based value proxy, the point size reflects run count, and the quadrants separate workflows into `keep`, `optimize`, `simplify`, and `review`. + +This is the fastest way to answer: which workflows deserve investment, which should be simplified, and which demand a maintainer decision? + +### 4. Workflow Overlap Matrix + +This plot is a workflow-by-workflow similarity heatmap. It is intended to serve the role that people often imagine for a Venn diagram, but in a form that scales beyond two or three workflows. Similarity can be inferred from task domains, naming, trigger or schedule similarity, behavioral fingerprints, and repeated overlap signals from recent runs. + +This is the fastest way to answer: which workflows may be solving the same problem closely enough to justify consolidation review? + +The intended visuals are: + +- an episode risk-cost frontier showing which execution chains are both expensive and risky +- a workflow stability matrix showing which workflows repeatedly accumulate control, risk, or fallback problems +- a repository portfolio map that separates `keep`, `optimize`, `simplify`, and `review` candidates +- a workflow overlap matrix that acts like a portfolio overlap view, similar in purpose to a Venn diagram but usually more precise for many workflows + +This format is better than a purely textual report when maintainers need to answer questions such as: + +- which workflows sit on the cost-risk frontier +- which workflows are stable but overbuilt +- which workflows cluster together strongly enough to deserve consolidation review +- which problems are isolated incidents versus repeated patterns across the repository + +The goal is not visual novelty for its own sake. Each chart should support a decision. In practice, the overlap matrix supports consolidation review, the portfolio map supports prioritization, and the risk-cost frontier supports immediate optimization work. + +The most useful reading of those plots is outcome-adjusted rather than usage-only. Cost is more informative when read as cost per successful run. Token volume is more informative when read as effective tokens per successful run. Tool overhead is more informative when read as median tool calls or turns per successful run rather than raw totals. + +## Metric glossary + +The visual report uses a small number of derived metrics to keep the plots decision-oriented. + +`episode_risk_score` +This is a composite risk score for a single execution episode. It combines risky nodes, poor-control nodes, MCP failures, blocked requests, repeated regression markers, and escalation eligibility. It exists to answer one question quickly: which episodes combine multiple warning signals at once? + +`workflow_instability_score` +This is a workflow-level instability score derived from repeated risky runs, poor-control assessments, resource-heavy assessments, latest-success fallback usage, blocked requests, and MCP failures. It exists to separate chronic instability from one-off incidents. + +`workflow_value_proxy` +This is a repository-local proxy for workflow value. It is not a business KPI. It combines successful recent usage, stability, repeat use, and the absence of strong overkill or reduction signals. It exists to help rank workflows into `keep`, `optimize`, `simplify`, and `review` rather than to claim objective business value. + +`workflow_overlap_score` +This is an approximate similarity score between two workflows. It blends task domain, trigger or schedule similarity, naming, behavioral fingerprints, and assessment patterns. It exists to support consolidation review, not to prove duplication with mathematical certainty. + +`cost per successful run` +This is the preferred cost view when enough successful runs exist. It is more decision-useful than raw spend because it separates expensive-but-effective workflows from expensive-and-unreliable workflows. + +`effective tokens per successful run` +This is the preferred token-efficiency view when comparing routes or workflows across models. It is more useful than raw token totals because it accounts for token class weighting and model differences. + +## Calibration from a real repository sample + +In this repository, a live sample of recent runs produced three practical calibration lessons for the kit. + +First, estimated dollar cost was sparse. Effective Tokens carried much more of the usable efficiency signal across workflows. In repositories with similar telemetry, the portfolio map should switch its primary x-axis from recent cost to Effective Tokens, or Effective Tokens per successful run when enough successful runs exist. + +Second, task-domain coverage was coarse. Most sampled runs landed in `general_automation`, while the behavior fingerprints still separated them into meaningful groups such as directed lean runs, exploratory heavy runs, and adaptive moderate runs. In repositories with similar distributions, the portfolio analysis should compare workflows by behavior cluster and workflow family when the domain layer is too coarse to be reliable. + +Third, `partially_reducible` appeared often enough that it should be treated as a repeated reduction hint, not an automatic negative verdict. In practice, that signal becomes most useful when it appears together with high Effective Tokens, high turn counts, or repeated resource-heavy behavior. + +Those findings do not replace the general design. They make the default interpretation more robust when repositories have sparse cost fields, weak domain separation, or many exploratory workflows. + +## Domain-specific reading + +The same signals do not mean the same thing for every workflow type. + +For `triage`, `repo_maintenance`, and `issue_response`, the most valuable question is whether the workflow is too agentic for its job. These domains are the strongest candidates for deterministic replacements, smaller models, and narrow-tool routing. + +For `research`, broader tool use and exploration can be justified, but repeated cost still needs evidence of value. In practice, the most important question is often whether data-gathering work should move into deterministic pre-steps while the agent keeps only the analytical core. + +For `code_fix`, higher cost may be justified when successful write actions are intentional and controlled. The most important question is usually not pure spend, but whether the workflow combines cost with instability, blocked requests, or weak control. + +For `release_ops`, reliability dominates. The most important question is whether the workflow is stable, repeatable, and well controlled; moderate cost is often acceptable, repeated instability is not. + +For delegated workflows, episode-level interpretation matters more than local workflow-level interpretation. A worker that looks expensive in isolation may still be justified inside a coherent larger execution chain. + +This is why the kit should compare workflows within similar task domains first. A cheap triage workflow and an expensive research workflow are not automatically substitutes for each other. + +### Report form + +The most useful discussion form is: + +1. `Executive Summary` for the overall decision. +2. `Key Metrics` for repository-level scale. +3. `Highest Risk Episodes` and `Episode Regressions` for operational findings. +4. `Visual Diagnostics` with the four charts in the fixed order above. +5. `Portfolio Opportunities` for repository-level cleanup candidates. +6. `Recommended Actions` for the final ranked decisions. + +Under each chart, the report should include two short blocks: `Decision` and `Why it matters`. That keeps the visuals analytical instead of decorative. + +## Why it matters for COGS reduction + +The kit is useful for COGS reduction because it turns agentic workflow spend into a reviewable operational signal instead of a vague complaint about “expensive runs”. In practice it helps surface four common sources of waste. + +First, it highlights workflows that are too expensive for the task domain they serve. A workflow that is consistently resource-heavy, repeatedly compared against `latest_success`, or marked as overkill for agentic execution is a candidate for a smaller model, tighter prompts, or deterministic automation. + +Second, it exposes avoidable control failures. Repeated blocked requests, MCP failures, or poor-control assessments often mean the system is spending tokens and Actions minutes on retries, fallback behavior, or incomplete execution paths rather than useful work. + +Third, it makes orchestration costs legible. Episode rollups prevent distributed workflows from hiding their true aggregate cost across many child runs. This is important for repositories that dispatch workers or chain `workflow_run` triggers. + +Fourth, it gives maintainers a way to prioritize optimization work. The escalation logic is designed to avoid one issue per workflow. Instead, it groups repeated problems into a single actionable report so repository owners can focus on the highest-value fixes first. + +Fifth, it helps maintainers spot workflows that may no longer deserve their current shape. When a workflow is repeatedly overkill for its domain, rarely active, or plausibly overlapping with another workflow, the kit can surface that as a cleanup opportunity before the repository accumulates more operational drag. + +## Accuracy and cost caveats + +The kit is accurate as an observability and optimization tool, but its cost signals are not equivalent to billing records. + +`action_minutes` is an estimate derived from workflow duration and rounded to billable minutes. It is useful for relative comparison and trend detection, but it does not represent a GitHub invoice line item. + +`estimated_cost` is only as authoritative as the engine logs that produced it. For some engines, the value comes from structured log fields emitted by the runtime. For portfolio analysis and prioritization this is usually sufficient, but the number should still be treated as a run-level estimate rather than finance-grade accounting. + +Effective Tokens are also intentionally not a billing unit. They are a normalization layer that makes cross-run and cross-model comparisons more useful. Use them to answer “which workflows are inefficient?” rather than “what exact amount will appear on the invoice?” + +## When to use it + +This pattern is a good fit when: + +- A repository has multiple agentic workflows and maintainers need a weekly operational summary. +- Orchestrated workflows make per-run analysis misleading. +- The team wants an evidence-based way to identify model downgrades, prompt tightening, deterministic replacements, or workflow cleanup candidates. +- The repository already uses `gh aw logs` and `gh aw audit` for investigation and wants the same signals in an automated report. + +This pattern is a poor fit when a repository has only one low-frequency workflow or when exact billing reconciliation is the primary requirement. + +For organization-wide or enterprise-wide deployment, it is also a poor fit as a direct copy-paste workflow if there is no central repository, no cross-repository token strategy, or no clear allowlist of repositories to observe. + +## Relationship to other tools + +The kit does not replace the lower-level debugging tools. + +- Use [`gh aw logs`](/gh-aw/reference/audit/#gh-aw-logs---format-fmt) to inspect cross-run trends directly. +- Use [`gh aw audit`](/gh-aw/reference/audit/#gh-aw-audit-run-id-or-url) for a detailed single-run report. +- Use [Cost Management](/gh-aw/reference/cost-management/) to understand Actions minutes, inference spend, and optimization levers. +- Use [Cross-Repository Operations](/gh-aw/reference/cross-repository/) and [MultiRepoOps](/gh-aw/patterns/multi-repo-ops/) when the observability workflow needs to read or coordinate across multiple repositories. + +The Agentic Observability Kit sits above those tools. It is the scheduled reviewer that turns those raw signals into one repository-level report. + +## Portfolio review capabilities + +The kit now includes the most useful repository-level capabilities that were previously split into a separate portfolio-style review. + +In this repository, the standalone `portfolio-analyst` workflow has been superseded by the kit. Maintainers who want one weekly report should use the Agentic Observability Kit instead of running both workflows. + +At repository scope, that means the report can now surface questions such as: + +- which workflows appear repeatedly overkill for their task domain +- which workflows look expensive relative to their recent value or stability +- which workflows may overlap in purpose, trigger pattern, or schedule +- which workflows look stale or weakly justified by recent activity + +The highest-value cleanup candidates are usually workflows that are dominated on more than one axis at the same time, such as expensive and unstable, cheap but consistently low-value, or overlapping and weaker than a nearby alternative. + +This remains an evidence-based repository-maintainer view, not a full fleet-governance system. The primary job of the kit is still operational observability. + +At organization or enterprise scope, the same pattern can be extended into a deeper portfolio rollup. That broader layer is where consolidation, retirement, cross-repository overlap, and fleet-wide prioritization fit best. + +> [!TIP] +> The repository-level kit is now sufficient for maintainers who want one weekly report covering both operational regressions and evidence-based repository portfolio cleanup opportunities. Central rollups remain the better place for full portfolio governance. + +## Source workflow + +The built-in workflow lives at [`/.github/workflows/agentic-observability-kit.md`](https://github.com/github/gh-aw/blob/main/.github/workflows/agentic-observability-kit.md). + +> [!NOTE] +> The workflow prompt prefers deterministic episode data over prompt-time reconstruction. If episode data is missing or incomplete, the report is expected to call that out as an observability finding rather than silently guessing. diff --git a/docs/src/content/docs/patterns/central-repo-ops.mdx b/docs/src/content/docs/patterns/central-repo-ops.mdx index 690b78cbe29..bcd7c80b5a9 100644 --- a/docs/src/content/docs/patterns/central-repo-ops.mdx +++ b/docs/src/content/docs/patterns/central-repo-ops.mdx @@ -3,10 +3,7 @@ title: CentralRepoOps description: Operate and roll out changes across many repositories from a single private control repository. --- -> [!WARNING] -> **Experimental:** CentralRepoOps is still experimental! Things may break, change, or be removed without deprecation at any time. - -CentralRepoOps is a [MultiRepoOps](/gh-aw/patterns/multi-repo-ops/) deployment variant where a single private repository acts as a control plane for large-scale operations across many repositories. +CentralRepoOps uses a single private repository as a control plane for large-scale operations across many repositories. Use this pattern for organization-wide rollouts, phased adoption (pilot waves first), central governance, and security-aware prioritization across tens or hundreds of repositories. Each orchestrator run delivers consistent policy gates, controlled fan-out (`max`), and a complete decision trail — without pushing `main` changes to individual target repositories.