Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/audit-workflows.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ Use the agentic-workflows MCP tool `logs` with parameters:
Output is saved to: /tmp/gh-aw/aw-mcp/logs
```

**Engine Classification**: Use `summary.engine_counts` from the `logs` tool output to report engine usage. Each run also has an `agent` field (e.g., `"copilot"`, `"claude"`, `"codex"`). Both are derived from the `engine_id` field in `aw_info.json`, which is the authoritative source for engine type.

**IMPORTANT**: Do NOT infer engine type by scanning `.lock.yml` files. Lock files contain the word `copilot` in allowed-domains lists and workflow source paths regardless of which engine the workflow uses, causing false positives.

**Analyze**: Review logs for:
- Missing tools (patterns, frequency, legitimacy)
- Errors (tool execution, MCP failures, auth, timeouts, resources)
Expand Down
8 changes: 8 additions & 0 deletions .mcp.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"servers": {
"github-agentic-workflows": {
"command": "gh",
"args": ["aw", "mcp-server"]
}
}
}
17 changes: 17 additions & 0 deletions pkg/cli/logs_report.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@ type LogsSummary struct {
TotalEpisodes int `json:"total_episodes" console:"header:Total Episodes"`
HighConfidenceEpisodes int `json:"high_confidence_episodes" console:"header:High Confidence Episodes"`
TotalGitHubAPICalls int `json:"total_github_api_calls,omitempty" console:"header:Total GitHub API Calls,format:number,omitempty"`
// EngineCounts maps engine_id (from aw_info.json) to the number of runs using that engine.
// Use this field to accurately classify engine types — do NOT infer engines by scanning
// lock files, which contain the word "copilot" in allowed-domains and workflow-source paths
// regardless of which engine the workflow actually uses.
EngineCounts map[string]int `json:"engine_counts,omitempty" console:"-"`
}

// RunData contains information about a single workflow run
Expand Down Expand Up @@ -134,6 +139,11 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation
var totalMissingData int
var totalSafeItems int
var totalGitHubAPICalls int
// engineCounts tracks the number of runs per engine_id, sourced from aw_info.json.
// This is the authoritative engine classification — do not infer engine type from
// lock file contents, which contain "copilot" in allowed-domains and source paths
// regardless of which engine the workflow uses.
engineCounts := make(map[string]int)
Copy link

Copilot AI Apr 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

engineCounts is allocated unconditionally even when no runs have aw_info.json/engine_id. This is a minor but easy-to-avoid allocation: declare it as var engineCounts map[string]int and initialize it lazily the first time an agentID is encountered (and keep the existing len(engineCounts) > 0 guard, since len(nil) is 0).

Suggested change
engineCounts := make(map[string]int)
var engineCounts map[string]int

Copilot uses AI. Check for mistakes.

// Build runs data
// Initialize as empty slice to ensure JSON marshals to [] instead of null
Expand Down Expand Up @@ -175,6 +185,10 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation
if awContext == nil {
awContext = pr.AwContext
}
// Accumulate engine counts from aw_info.json data (authoritative source).
if agentID != "" {
engineCounts[agentID]++
}

comparison := buildAuditComparisonForProcessedRuns(pr, processedRuns)

Expand Down Expand Up @@ -255,6 +269,9 @@ func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation
TotalSafeItems: totalSafeItems,
TotalGitHubAPICalls: totalGitHubAPICalls,
}
if len(engineCounts) > 0 {
summary.EngineCounts = engineCounts
}

episodes, edges := buildEpisodeData(runs, processedRuns)
for _, episode := range episodes {
Expand Down
52 changes: 52 additions & 0 deletions pkg/cli/logs_report_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
package cli

import (
"os"
"path/filepath"
"testing"
"time"
)
Expand Down Expand Up @@ -908,3 +910,53 @@ func TestDeriveRunClassification(t *testing.T) {
})
}
}

// TestBuildLogsDataEngineCountsFromAwInfo verifies that engine_counts in the summary
// is populated from aw_info.json data (the authoritative engine source), not from
// lock file string matching.
func TestBuildLogsDataEngineCountsFromAwInfo(t *testing.T) {
createRunDir := func(engineID string) string {
dir := t.TempDir()
awInfo := `{"engine_id":"` + engineID + `","engine_name":"Test","workflow_name":"test","created_at":"2024-01-01T00:00:00Z"}`
if err := os.WriteFile(filepath.Join(dir, "aw_info.json"), []byte(awInfo), 0600); err != nil {
t.Fatalf("Failed to write aw_info.json: %v", err)
}
Comment on lines +919 to +923
Copy link

Copilot AI Apr 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test constructs JSON via string concatenation, which can produce invalid JSON if engineID contains characters needing escaping (quotes, backslashes, etc.). Consider building an AwInfo (or a small struct with engine_id) and marshaling it with encoding/json to make the fixture generation robust and easier to extend.

Copilot uses AI. Check for mistakes.
return dir
}

claudeDir := createRunDir("claude")
claudeDir2 := createRunDir("claude")
copilotDir := createRunDir("copilot")

processedRuns := []ProcessedRun{
{Run: WorkflowRun{DatabaseID: 1, WorkflowName: "wf-claude-1", LogsPath: claudeDir}},
{Run: WorkflowRun{DatabaseID: 2, WorkflowName: "wf-claude-2", LogsPath: claudeDir2}},
{Run: WorkflowRun{DatabaseID: 3, WorkflowName: "wf-copilot", LogsPath: copilotDir}},
}

data := buildLogsData(processedRuns, "/tmp/logs", nil)
Copy link

Copilot AI Apr 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using a hard-coded /tmp/logs makes the test less portable (notably on Windows) and can couple it to a specific filesystem layout. Prefer using t.TempDir() (or deriving a temp output directory via filepath.Join(t.TempDir(), ...)) for the outputDir argument.

Suggested change
data := buildLogsData(processedRuns, "/tmp/logs", nil)
outputDir := t.TempDir()
data := buildLogsData(processedRuns, outputDir, nil)

Copilot uses AI. Check for mistakes.

if data.Summary.EngineCounts == nil {
t.Fatal("EngineCounts should not be nil when runs have aw_info.json")
}
if got := data.Summary.EngineCounts["claude"]; got != 2 {
t.Errorf("Expected 2 claude runs, got %d", got)
}
if got := data.Summary.EngineCounts["copilot"]; got != 1 {
t.Errorf("Expected 1 copilot run, got %d", got)
}
// Verify individual RunData.Agent fields also reflect the engine from aw_info.json
agentsByID := make(map[int64]string)
for _, run := range data.Runs {
agentsByID[run.DatabaseID] = run.Agent
}
if agentsByID[1] != "claude" {
t.Errorf("Run 1: expected agent=claude, got %q", agentsByID[1])
}
if agentsByID[2] != "claude" {
t.Errorf("Run 2: expected agent=claude, got %q", agentsByID[2])
}
if agentsByID[3] != "copilot" {
t.Errorf("Run 3: expected agent=copilot, got %q", agentsByID[3])
}
}
Loading