From d79f8c21a955d9231bf88433eff62d767f336942 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 03:52:34 +0000 Subject: [PATCH 1/3] Initial plan From e09317b667300b00c36d9d3464bd4731eb6ab388 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 04:00:55 +0000 Subject: [PATCH 2/3] Initial plan: add engine counts to LogsSummary and update audit workflow Agent-Logs-Url: https://github.com/github/gh-aw/sessions/5912b901-d9a8-4f45-aae8-4e6671fe4594 Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .mcp.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .mcp.json diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 00000000000..96e7285c803 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,11 @@ +{ + "servers": { + "github-agentic-workflows": { + "command": "gh", + "args": [ + "aw", + "mcp-server" + ] + } + } +} \ No newline at end of file From 7d352a5e3dfbcb18499166164b664b325217ad35 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 04:16:32 +0000 Subject: [PATCH 3/3] fix: add engine_counts to LogsSummary from aw_info.json to fix false positive engine classification Agent-Logs-Url: https://github.com/github/gh-aw/sessions/5912b901-d9a8-4f45-aae8-4e6671fe4594 Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/audit-workflows.md | 4 +++ .mcp.json | 7 ++-- pkg/cli/logs_report.go | 17 +++++++++ pkg/cli/logs_report_test.go | 52 ++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/.github/workflows/audit-workflows.md b/.github/workflows/audit-workflows.md index 9a6590c3859..51cf4a2e89c 100644 --- a/.github/workflows/audit-workflows.md +++ b/.github/workflows/audit-workflows.md @@ -66,6 +66,10 @@ Use the agentic-workflows MCP tool `logs` with parameters: Output is saved to: /tmp/gh-aw/aw-mcp/logs ``` +**Engine Classification**: Use `summary.engine_counts` from the `logs` tool output to report engine usage. Each run also has an `agent` field (e.g., `"copilot"`, `"claude"`, `"codex"`). Both are derived from the `engine_id` field in `aw_info.json`, which is the authoritative source for engine type. + +**IMPORTANT**: Do NOT infer engine type by scanning `.lock.yml` files. Lock files contain the word `copilot` in allowed-domains lists and workflow source paths regardless of which engine the workflow uses, causing false positives. + **Analyze**: Review logs for: - Missing tools (patterns, frequency, legitimacy) - Errors (tool execution, MCP failures, auth, timeouts, resources) diff --git a/.mcp.json b/.mcp.json index 96e7285c803..01021df64d9 100644 --- a/.mcp.json +++ b/.mcp.json @@ -2,10 +2,7 @@ "servers": { "github-agentic-workflows": { "command": "gh", - "args": [ - "aw", - "mcp-server" - ] + "args": ["aw", "mcp-server"] } } -} \ No newline at end of file +} diff --git a/pkg/cli/logs_report.go b/pkg/cli/logs_report.go index 09d21255525..2f3372b701e 100644 --- a/pkg/cli/logs_report.go +++ b/pkg/cli/logs_report.go @@ -67,6 +67,11 @@ type LogsSummary struct { TotalEpisodes int `json:"total_episodes" console:"header:Total Episodes"` HighConfidenceEpisodes int `json:"high_confidence_episodes" console:"header:High Confidence Episodes"` TotalGitHubAPICalls int `json:"total_github_api_calls,omitempty" console:"header:Total GitHub API Calls,format:number,omitempty"` + // EngineCounts maps engine_id (from aw_info.json) to the number of runs using that engine. + // Use this field to accurately classify engine types — do NOT infer engines by scanning + // lock files, which contain the word "copilot" in allowed-domains and workflow-source paths + // regardless of which engine the workflow actually uses. + EngineCounts map[string]int `json:"engine_counts,omitempty" console:"-"` } // RunData contains information about a single workflow run @@ -134,6 +139,11 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation var totalMissingData int var totalSafeItems int var totalGitHubAPICalls int + // engineCounts tracks the number of runs per engine_id, sourced from aw_info.json. + // This is the authoritative engine classification — do not infer engine type from + // lock file contents, which contain "copilot" in allowed-domains and source paths + // regardless of which engine the workflow uses. + engineCounts := make(map[string]int) // Build runs data // Initialize as empty slice to ensure JSON marshals to [] instead of null @@ -175,6 +185,10 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation if awContext == nil { awContext = pr.AwContext } + // Accumulate engine counts from aw_info.json data (authoritative source). + if agentID != "" { + engineCounts[agentID]++ + } comparison := buildAuditComparisonForProcessedRuns(pr, processedRuns) @@ -255,6 +269,9 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation TotalSafeItems: totalSafeItems, TotalGitHubAPICalls: totalGitHubAPICalls, } + if len(engineCounts) > 0 { + summary.EngineCounts = engineCounts + } episodes, edges := buildEpisodeData(runs, processedRuns) for _, episode := range episodes { diff --git a/pkg/cli/logs_report_test.go b/pkg/cli/logs_report_test.go index fa1fa3d93d6..994b41a7263 100644 --- a/pkg/cli/logs_report_test.go +++ b/pkg/cli/logs_report_test.go @@ -3,6 +3,8 @@ package cli import ( + "os" + "path/filepath" "testing" "time" ) @@ -908,3 +910,53 @@ func TestDeriveRunClassification(t *testing.T) { }) } } + +// TestBuildLogsDataEngineCountsFromAwInfo verifies that engine_counts in the summary +// is populated from aw_info.json data (the authoritative engine source), not from +// lock file string matching. +func TestBuildLogsDataEngineCountsFromAwInfo(t *testing.T) { + createRunDir := func(engineID string) string { + dir := t.TempDir() + awInfo := `{"engine_id":"` + engineID + `","engine_name":"Test","workflow_name":"test","created_at":"2024-01-01T00:00:00Z"}` + if err := os.WriteFile(filepath.Join(dir, "aw_info.json"), []byte(awInfo), 0600); err != nil { + t.Fatalf("Failed to write aw_info.json: %v", err) + } + return dir + } + + claudeDir := createRunDir("claude") + claudeDir2 := createRunDir("claude") + copilotDir := createRunDir("copilot") + + processedRuns := []ProcessedRun{ + {Run: WorkflowRun{DatabaseID: 1, WorkflowName: "wf-claude-1", LogsPath: claudeDir}}, + {Run: WorkflowRun{DatabaseID: 2, WorkflowName: "wf-claude-2", LogsPath: claudeDir2}}, + {Run: WorkflowRun{DatabaseID: 3, WorkflowName: "wf-copilot", LogsPath: copilotDir}}, + } + + data := buildLogsData(processedRuns, "/tmp/logs", nil) + + if data.Summary.EngineCounts == nil { + t.Fatal("EngineCounts should not be nil when runs have aw_info.json") + } + if got := data.Summary.EngineCounts["claude"]; got != 2 { + t.Errorf("Expected 2 claude runs, got %d", got) + } + if got := data.Summary.EngineCounts["copilot"]; got != 1 { + t.Errorf("Expected 1 copilot run, got %d", got) + } + // Verify individual RunData.Agent fields also reflect the engine from aw_info.json + agentsByID := make(map[int64]string) + for _, run := range data.Runs { + agentsByID[run.DatabaseID] = run.Agent + } + if agentsByID[1] != "claude" { + t.Errorf("Run 1: expected agent=claude, got %q", agentsByID[1]) + } + if agentsByID[2] != "claude" { + t.Errorf("Run 2: expected agent=claude, got %q", agentsByID[2]) + } + if agentsByID[3] != "copilot" { + t.Errorf("Run 3: expected agent=copilot, got %q", agentsByID[3]) + } +}