diff --git a/pkg/workflow/schemas/github-workflow.json b/pkg/workflow/schemas/github-workflow.json index afa39b90149..e3437f83ba5 100644 --- a/pkg/workflow/schemas/github-workflow.json +++ b/pkg/workflow/schemas/github-workflow.json @@ -62,16 +62,16 @@ } ] }, - "container": { + "jobContainer": { "type": "object", "properties": { "image": { - "$comment": "https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions#jobsjob_idcontainerimage", + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idcontainerimage", "description": "The Docker image to use as the container to run the action. The value can be the Docker Hub image name or a registry name.", "type": "string" }, "credentials": { - "$comment": "https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontainercredentials", + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idcontainercredentials", "description": "If the image's container registry requires authentication to pull the image, you can use credentials to set a map of the username and password. The credentials are the same values that you would provide to the `docker login` command.", "type": "object", "properties": { @@ -84,12 +84,12 @@ } }, "env": { - "$comment": "https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions#jobsjob_idcontainerenv", + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idcontainerenv", "$ref": "#/definitions/env", - "description": "Sets an array of environment variables in the container." + "description": "Sets a map of environment variables in the container." }, "ports": { - "$comment": "https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions#jobsjob_idcontainerports", + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idcontainerports", "description": "Sets an array of ports to expose on the container.", "type": "array", "items": { @@ -105,7 +105,7 @@ "minItems": 1 }, "volumes": { - "$comment": "https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions#jobsjob_idcontainervolumes", + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idcontainervolumes", "description": "Sets an array of volumes for the container to use. You can use volumes to share data between services or other steps in a job. You can specify named Docker volumes, anonymous Docker volumes, or bind mounts on the host.\nTo specify a volume, you specify the source and destination path: :\nThe is a volume name or an absolute path on the host machine, and is an absolute path in the container.", "type": "array", "items": { @@ -114,7 +114,7 @@ "minItems": 1 }, "options": { - "$comment": "https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions#jobsjob_idcontaineroptions", + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idcontaineroptions", "description": "Additional Docker container resource options. For a list of options, see https://docs.docker.com/engine/reference/commandline/create/#options.", "type": "string" } @@ -122,6 +122,76 @@ "required": ["image"], "additionalProperties": false }, + "serviceContainer": { + "type": "object", + "properties": { + "image": { + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idservicesservice_idimage", + "description": "The Docker image to use as the service container to run the action. The value can be the Docker Hub image name or a registry name.", + "type": "string" + }, + "credentials": { + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idservicesservice_idcredentials", + "description": "If the image's container registry requires authentication to pull the image, you can use credentials to set a map of the username and password. The credentials are the same values that you would provide to the `docker login` command.", + "type": "object", + "properties": { + "username": { + "type": "string" + }, + "password": { + "type": "string" + } + } + }, + "env": { + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idservicesservice_idenv", + "$ref": "#/definitions/env", + "description": "Sets a map of environment variables in the service container." + }, + "ports": { + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idservicesservice_idports", + "description": "Sets an array of ports to expose on the service container.", + "type": "array", + "items": { + "oneOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "minItems": 1 + }, + "volumes": { + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idservicesservice_idvolumes", + "description": "Sets an array of volumes for the service container to use. You can use volumes to share data between services or other steps in a job. You can specify named Docker volumes, anonymous Docker volumes, or bind mounts on the host.\nTo specify a volume, you specify the source and destination path: :\nThe is a volume name or an absolute path on the host machine, and is an absolute path in the container.", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "options": { + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idservicesservice_idoptions", + "description": "Additional Docker container resource options. For a list of options, see https://docs.docker.com/engine/reference/commandline/create/#options.", + "type": "string" + }, + "command": { + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idservicesservice_idcommand", + "description": "Overrides the Docker image's default command (`CMD`). The value is passed as arguments after the image name in the `docker create` command. If you also specify `entrypoint`, `command` provides the arguments to that entrypoint.", + "type": "string" + }, + "entrypoint": { + "$comment": "https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idservicesservice_identrypoint", + "description": "Overrides the Docker image's default `ENTRYPOINT`. The value is a single string defining the executable to run. Use this when you need to replace the image's entrypoint entirely. You can combine `entrypoint` with `command` to pass arguments to the custom entrypoint.", + "type": "string" + } + }, + "required": ["image"], + "additionalProperties": false + }, "defaults": { "type": "object", "properties": { @@ -833,7 +903,7 @@ "type": "string" }, { - "$ref": "#/definitions/container" + "$ref": "#/definitions/jobContainer" } ] }, @@ -842,7 +912,7 @@ "description": "Additional containers to host services for a job in a workflow. These are useful for creating databases or cache services like redis. The runner on the virtual machine will automatically create a network and manage the life cycle of the service containers.\nWhen you use a service container for a job or your step uses container actions, you don't need to set port information to access the service. Docker automatically exposes all ports between containers on the same network.\nWhen both the job and the action run in a container, you can directly reference the container by its hostname. The hostname is automatically mapped to the service name.\nWhen a step does not use a container action, you must access the service using localhost and bind the ports.", "type": "object", "additionalProperties": { - "$ref": "#/definitions/container" + "$ref": "#/definitions/serviceContainer" } }, "concurrency": { diff --git a/scratchpad/expression-consolidation.md b/scratchpad/expression-consolidation.md new file mode 100644 index 00000000000..02beddd985b --- /dev/null +++ b/scratchpad/expression-consolidation.md @@ -0,0 +1,283 @@ +# Pre-Consolidation Analysis: `pkg/workflow` Expression Subsystem + +## Overview + +The expression subsystem spans **9 production files** and **7+ test files** in `pkg/workflow/`. +The consolidation target is `copilot/consolidate-optimization-work`. + +--- + +## File Inventory + +### Production Files + +| File | Lines | Responsibility | +|------|------:|----------------| +| `expression_nodes.go` | 196 | AST node types (`ConditionNode`, `AndNode`, `OrNode`, `NotNode`, `DisjunctionNode`, `FunctionCallNode`, `PropertyAccessNode`, `StringLiteralNode`, `BooleanLiteralNode`, `ComparisonNode`) | +| `expression_builder.go` | 291 | High-level `Build*` factory functions + `RenderCondition`, `RenderConditionAsIf` | +| `expression_optimizer.go` | 541 | Boolean-algebra optimizer (`OptimizeExpression`), called by `RenderCondition` | +| `expression_parser.go` | 510 | Recursive-descent parser (`ParseExpression`, `ExpressionParser`) | +| `expression_patterns.go` | 191 | **Centralized** public regex vars (`ExpressionPattern`, `NeedsStepsPattern`, `OrPattern`, etc.) | +| `expression_extraction.go` | 366 | `ExpressionExtractor`, env-var substitution, `SubstituteImportInputs` | +| `expression_safety_validation.go` | 302 | Security allowlist validation, **private** duplicate regexes | +| `expression_syntax_validation.go` | 236 | Structural validation (balanced braces, quotes, parentheses), **private** duplicate regexes | +| `known_needs_expressions.go` | 263 | `generateKnownNeedsExpressions`, `filterExpressionsForActivation`, `parseNeedsField` | +| **Total** | **2,896** | | + +### Test Files + +| File | Lines | +|------|------:| +| `expression_optimizer_test.go` | 1,210 | +| `expressions_test.go` | 1,144 | +| `expression_safety_test.go` | 849 | +| `expression_parser_comprehensive_test.go` | 621 | +| `expression_coverage_test.go` | 225 | +| `expressions_benchmark_test.go` | 183 | +| `expression_extraction_test.go` | ~200 | +| `expression_patterns_test.go` | ~100 | +| **Total** | **~4,532+** | + +--- + +## Key Finding: Regex Duplication + +`expression_patterns.go` was created to be the **single source of truth** for all expression regex +patterns. However, two files still carry **private duplicates** of patterns already defined there. + +### Duplication Map — `expression_safety_validation.go` + +| `expression_patterns.go` (public) | `expression_safety_validation.go` (private duplicate) | +|-----------------------------------|-------------------------------------------------------| +| `ExpressionPatternDotAll` | `expressionRegex` | +| `NeedsStepsPattern` | `needsStepsRegex` | +| `InputsPattern` | `inputsRegex` | +| `WorkflowCallInputsPattern` | `workflowCallInputsRegex` | +| `AWInputsPattern` | `awInputsRegex` | +| `AWImportInputsPattern` | `awImportInputsRegex` | +| `EnvPattern` | `envRegex` | +| `ComparisonExtractionPattern` | `comparisonExtractionRegex` | +| `OrPattern` | `orExpressionPattern` | + +### Duplication Map — `expression_syntax_validation.go` + +| `expression_patterns.go` (public) | `expression_syntax_validation.go` (private duplicate) | +|-----------------------------------|-------------------------------------------------------| +| `StringLiteralPattern` | `stringLiteralRegex` | +| `NumberLiteralPattern` | `numberLiteralRegex` | + +### Unique Patterns in `expression_syntax_validation.go` (not yet centralized) + +| Private name | Regex | Candidate public name | +|--------------|-------|-----------------------| +| `expressionBracesPattern` | `` `\$\{\{([^}]*)\}\}` `` | `ExpressionBracesPattern` | +| `exprPartSplitRe` | `` `[.\[\]]+` `` | `ExpressionPartSplitPattern` | +| `exprNumericPartRe` | `` `^\d+$` `` | `ExpressionNumericPartPattern` | + +**Summary**: 9 regexes in `expression_safety_validation.go` and 2 regexes in +`expression_syntax_validation.go` duplicate patterns already in `expression_patterns.go`. +This means **double compilation overhead at startup** and **two sources of truth** that can drift. + +--- + +## Regex Pattern Cross-Reference (Exact Strings) + +### `ExpressionPatternDotAll` vs `expressionRegex` + +``` +expression_patterns.go: `(?s)\$\{\{(.*?)\}\}` +expression_safety_validation: `(?s)\$\{\{(.*?)\}\}` ← identical +``` + +### `NeedsStepsPattern` vs `needsStepsRegex` + +``` +expression_patterns.go: `^(needs|steps)\.[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)*$` +expression_safety_validation: `^(needs|steps)\.[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)*$` ← identical +``` + +### `ComparisonExtractionPattern` vs `comparisonExtractionRegex` + +``` +expression_patterns.go: `([a-zA-Z_][a-zA-Z0-9_.]*)\s*(?:==|!=|<|>|<=|>=)\s*` +expression_safety_validation: `([a-zA-Z_][a-zA-Z0-9_.]*)\s*(?:==|!=|<|>|<=|>=)\s*` ← identical +``` + +### `StringLiteralPattern` vs `stringLiteralRegex` + +``` +expression_patterns.go: `^'[^']*'$|^"[^"]*"$|^` + "`[^`]*`$" +expression_syntax_validation: `^'[^']*'$|^"[^"]*"$|^` + "`[^`]*`$" ← identical +``` + +### `NumberLiteralPattern` vs `numberLiteralRegex` + +``` +expression_patterns.go: `^-?\d+(\.\d+)?$` +expression_syntax_validation: `^-?\d+(\.\d+)?$` ← identical +``` + +--- + +## Dependency Graph + +``` +expression_nodes.go ← defines all AST types (no deps on other expression files) + ↑ +expression_builder.go ← Build* constructors, RenderCondition → calls OptimizeExpression + ↑ +expression_optimizer.go ← OptimizeExpression (pure tree transform, depends on nodes only) + ↑ +expression_parser.go ← ParseExpression → produces ConditionNode trees + +expression_patterns.go ← standalone regex vars, no deps on other expression files + +expression_extraction.go ← uses ExpressionPatternDotAll, NeedsStepsPattern, etc. + ALSO defines ExpressionMapping, ExpressionExtractor + +expression_safety_validation.go ← SHOULD use expression_patterns.go vars + CURRENTLY has 9 private duplicate regexes + +expression_syntax_validation.go ← SHOULD use expression_patterns.go vars + CURRENTLY has 2 private duplicate regexes + 3 unique + +known_needs_expressions.go ← uses ExpressionMapping from expression_extraction.go + produces []ExpressionMapping for the compiler +``` + +--- + +## Type Inventory + +### AST Node Types (`expression_nodes.go`) + +| Type | Kind | Description | +|------|------|-------------| +| `ConditionNode` | interface | Single method: `Render() string` | +| `ExpressionNode` | leaf | Raw expression string + optional description | +| `AndNode` | binary | `&&` operator | +| `OrNode` | binary | `\|\|` operator | +| `NotNode` | unary | `!` operator | +| `DisjunctionNode` | n-ary OR | Avoids deep nesting; supports multiline render | +| `FunctionCallNode` | leaf | `name(args...)` | +| `PropertyAccessNode` | leaf | Dotted path like `github.event_name` | +| `StringLiteralNode` | leaf | `'value'` | +| `BooleanLiteralNode` | leaf | `true` / `false` | +| `ComparisonNode` | binary | `left op right` where op ∈ `==`, `!=`, `<`, `>`, `<=`, `>=` | + +### Extractor Types (`expression_extraction.go`) + +| Type | Description | +|------|-------------| +| `ExpressionMapping` | Maps a `${{ expr }}` to an env-var name + metadata | +| `ExpressionExtractor` | Stateful extractor with dedup map | + +### Parser Types (`expression_parser.go`) + +| Type | Description | +|------|-------------| +| `ExpressionParser` | Recursive-descent parser | +| `token` / `tokenKind` | Lexer tokens (private) | + +### Validation Types (`expression_safety_validation.go`) + +| Type | Description | +|------|-------------| +| `ExpressionValidationOptions` | Options struct for `validateSingleExpression` | + +--- + +## Optimizer Rules (`expression_optimizer.go`, 541 lines) + +| Rule | Node type | Before → After | +|------|-----------|----------------| +| Constant folding | `NotNode` | `!true → false`, `!false → true` | +| Double negation | `NotNode` | `!!A → A` | +| Boolean identity (AND) | `AndNode` | `A && true → A` | +| Boolean identity (OR) | `OrNode` | `A \|\| false → A` | +| Boolean annihilation (AND) | `AndNode` | `A && false → false` | +| Boolean annihilation (OR) | `OrNode` | `A \|\| true → true` | +| Idempotent (AND) | `AndNode` | `A && A → A` | +| Idempotent (OR) | `OrNode` | `A \|\| A → A` | +| Complement (AND) | `AndNode` | `A && !A → false` | +| Complement (OR) | `OrNode` | `A \|\| !A → true` | +| De Morgan (AND) | `NotNode` | `!(A && B) → !A \|\| !B` | +| De Morgan (OR) | `NotNode` | `!(A \|\| B) → !A && !B` | +| Absorption (AND) | `AndNode` | `A && (A \|\| B) → A` | +| Absorption (OR) | `OrNode` | `A \|\| (A && B) → A` | +| Subsumption | `DisjunctionNode` | `disj(A, A&&B, …) → disj(A, …)` | +| Deduplication | `AndNode` / `DisjunctionNode` | removes identical terms | +| False-filtering | `DisjunctionNode` | removes `false` terms | +| True short-circuit | `DisjunctionNode` | entire disjunction → `true` | + +**Safety constraint**: No rule fires when a GitHub Actions status function +(`always()`, `success()`, `failure()`, `cancelled()`) appears in either operand. + +**Termination**: Fixed-point iteration, bounded by `maxOptimizationPasses = 10`. + +--- + +## Regex Compilation Count (startup cost) + +| File | Compiled regexes | Notes | +|------|----------------:|-------| +| `expression_patterns.go` | 20 | Public, centralized | +| `expression_safety_validation.go` | 9 | All duplicates of patterns above | +| `expression_syntax_validation.go` | 5 | 2 duplicates + 3 unique | +| `expression_extraction.go` | 4 | Not duplicated | +| **Total compiled** | **~38** | | +| **Total unique patterns** | **~27** | | +| **Wasted compilations** | **11** | | + +--- + +## Consolidation Plan + +### Step 1 — `expression_safety_validation.go`: replace 9 private vars + +| Remove | Replace with | +|--------|-------------| +| `expressionRegex` | `ExpressionPatternDotAll` | +| `needsStepsRegex` | `NeedsStepsPattern` | +| `inputsRegex` | `InputsPattern` | +| `workflowCallInputsRegex` | `WorkflowCallInputsPattern` | +| `awInputsRegex` | `AWInputsPattern` | +| `awImportInputsRegex` | `AWImportInputsPattern` | +| `envRegex` | `EnvPattern` | +| `comparisonExtractionRegex` | `ComparisonExtractionPattern` | +| `orExpressionPattern` | `OrPattern` | + +### Step 2 — `expression_syntax_validation.go`: replace 2 private vars + +| Remove | Replace with | +|--------|-------------| +| `stringLiteralRegex` | `StringLiteralPattern` | +| `numberLiteralRegex` | `NumberLiteralPattern` | + +### Step 3 — `expression_patterns.go`: promote 3 unique patterns (optional) + +| Add | Regex | Taken from | +|-----|-------|-----------| +| `ExpressionBracesPattern` | `` `\$\{\{([^}]*)\}\}` `` | `expression_syntax_validation.go` | +| `ExpressionPartSplitPattern` | `` `[.\[\]]+` `` | `expression_syntax_validation.go` | +| `ExpressionNumericPartPattern` | `` `^\d+$` `` | `expression_syntax_validation.go` | + +### Files changed + +| File | Action | +|------|--------| +| `expression_safety_validation.go` | Remove 9 private regex vars; update call sites | +| `expression_syntax_validation.go` | Remove 2 private regex vars; update call sites; optionally promote 3 unique patterns | +| `expression_patterns.go` | Optionally add 3 new public patterns | +| All `*_test.go` files | **No changes needed** | + +--- + +## Risk Assessment + +| Risk | Severity | Notes | +|------|----------|-------| +| Regex behavioral change | **None** | All duplicate patterns are byte-for-byte identical (verified above) | +| Startup perf change | **Positive** | From ~38 to ~27 compiled regexes | +| Test breakage | **None** | Internal implementation detail; tests use public API | +| Naming collision | **Low** | Names differ (`expressionRegex` vs `ExpressionPatternDotAll`) | diff --git a/scratchpad/observability-audit-consolidation.md b/scratchpad/observability-audit-consolidation.md new file mode 100644 index 00000000000..0732c3fd236 --- /dev/null +++ b/scratchpad/observability-audit-consolidation.md @@ -0,0 +1,200 @@ +# Pre-Consolidation Analysis: Observability & Audit Workflows + +## Scope + +Five workflows all overlap in the "monitor and report on agentic workflow runs" space: + +| Short name | File | Lines | +|------------|------|------:| +| **Kit** | `agentic-observability-kit.md` | 569 | +| **Audit** | `audit-workflows.md` | 99 | +| **DOR** | `daily-observability-report.md` | 393 | +| **WHM** | `workflow-health-manager.md` | 456 | +| **SOH** | `safe-output-health.md` | 361 | +| **Total** | | **1,878** | + +--- + +## Table 1 — What Each Workflow Currently Analyzes + +| Dimension | Kit | Audit | DOR | WHM | SOH | +|-----------|-----|-------|-----|-----|-----| +| **Primary question** | Are runs operationally healthy & portfolio optimal? | Were there issues, missing tools, or errors in the last 24 h? | Do firewall & MCP gateway logs provide adequate telemetry? | Are all 120+ workflows themselves structurally healthy? | Did safe-output jobs succeed or fail? | +| **Analysis subjects** | Workflow _runs_ (episodes) | Workflow _runs_ (raw logs) | Workflow _runs_ (log artifacts: access.log, gateway.jsonl) | Workflow _files_ + lock files + CI | Safe-output _jobs_ within runs | +| **Scope** | Full repository, all workflows | Full repository, all workflows | Full repository, firewall- and MCP-enabled runs only | Full repository, all `.md` + `.lock.yml` files | Full repository, safe-output job logs only | +| **Data model** | Episode graph (`episodes[]`, `edges[]`) + behavior fingerprint | Raw per-run logs from `logs` tool | File presence checks on log artifact directories | `git ls-files` + Actions API + repo-memory | Per-safe-output-job log lines | +| **What it looks for** | Risk regressions, cost outliers, portfolio overlap, escalation triggers | Missing tools, MCP failures, auth/timeout errors, performance | Missing `access.log`, missing `gateway.jsonl`/`rpc-messages.jsonl` | Missing lock files, failed runs, stale configs, dependency issues | Safe-output errors by type and cluster | +| **Time window (primary)** | 30 days (`count: 300–500`), emphasize last 14 | 24 hours (`start_date: "-1d"`) | 7 days (`start_date: "-7d"`) | 7 days per workflow (last 10 runs) | 24 hours (`start_date: "-1d"`) | +| **Time window (charts)** | 30 days (4 analytical charts) | 30 days (2 trend charts) | N/A (no charts) | 30 days (metrics from repo-memory) | N/A (no charts) | +| **Episode / lineage model** | ✅ Primary (`episodes[]`, `edges[]`) | ❌ Not used | ❌ Not used | ❌ Not used | ❌ Not used | +| **Firewall analysis** | ⚠️ Incidental (blocked requests as risk signal) | ⚠️ Incidental (blocked requests as error signal) | ✅ Primary (`access.log` coverage) | ❌ Not analyzed | ❌ Not analyzed | +| **MCP gateway analysis** | ⚠️ Incidental (MCP failure as risk signal) | ⚠️ Incidental (MCP failures as errors) | ✅ Primary (`gateway.jsonl`/`rpc-messages.jsonl`) | ❌ Not analyzed | ❌ Not analyzed | +| **Safe-output analysis** | ❌ Not analyzed | ❌ Not analyzed | ❌ Not analyzed | ⚠️ Incidental (safe-output job health) | ✅ Primary | +| **Portfolio / overlap** | ✅ Secondary appendix (workflow overlap matrix chart) | ❌ Not analyzed | ❌ Not analyzed | ⚠️ Incidental (stale/redundant detection) | ❌ Not analyzed | +| **Trend charts generated** | 4 (risk frontier, stability matrix, portfolio map, overlap matrix) | 2 (health trend, token/cost trend) | 0 | 0 | 0 | + +--- + +## Table 2 — Workflow Configuration Snapshot + +| Property | Kit | Audit | DOR | WHM | SOH | +|----------|-----|-------|-----|-----|-----| +| **Schedule** | Weekly (Mon ~08:00) | Daily | Daily | Daily | Daily | +| **Trigger** | schedule + `workflow_dispatch` | schedule + `workflow_dispatch` | `on: daily` | `on: daily` | schedule + `workflow_dispatch` | +| **Engine** | copilot | claude | codex | copilot | claude | +| **`strict: true`** | ✅ | ❌ | ✅ | ❌ | ✅ | +| **`timeout-minutes`** | 30 | 30 | 45 | 30 | 30 | +| **Tools: `agentic-workflows`** | ✅ | ✅ | ✅ | ❌ | ❌ (via import) | +| **Tools: `github`** | ✅ `[default, discussions]` | ❌ | ✅ `[default, discussions, actions]` | ✅ `[default, actions]` | ❌ | +| **Tools: `bash`** | ❌ | ❌ | ❌ | ✅ `[":*"]` | ❌ | +| **Tools: `edit`** | ❌ | ❌ | ❌ | ✅ | ❌ | +| **Tools: `repo-memory`** | ❌ | Via `repo-memory-standard` import | ❌ | ✅ (branch: `memory/meta-orchestrators`) | ❌ | +| **Tools: `cache-memory`** | ❌ | ❌ | ❌ | ❌ | ✅ | +| **`mount-as-clis: true`** | ✅ | ✅ | ✅ | ❌ | ❌ | +| **`features.mcp-cli`** | ✅ | ✅ | ✅ | ❌ | ❌ | +| **Permissions: `discussions`** | ✅ read | ❌ | ✅ read | ❌ | ❌ | +| **tracker-id** | `agentic-observability-kit` | `audit-workflows-daily` | `daily-observability-report` | _(none)_ | _(none)_ | + +--- + +## Table 3 — Output Artifacts + +| Output | Kit | Audit | DOR | WHM | SOH | +|--------|-----|-------|-----|-----|-----| +| **Discussion** | ✅ `[observability] …` expires 7d | ✅ `[audit-workflows] …` expires 1d | ✅ `[observability] …` expires 1d | ❌ | ✅ `[safe-output-health] …` expires 1d | +| **Escalation issue** | ✅ 1 max, `[observability escalation]`, labels: agentics/warning/observability | ❌ | ❌ | ✅ up to 10, `cookie` label, grouped | ❌ | +| **Comment on issue** | ❌ | ❌ | ❌ | ✅ up to 15 | ❌ | +| **Update issue** | ❌ | ❌ | ❌ | ✅ up to 5 | ❌ | +| **Charts (max)** | 4 PNG/SVG | 3 PNG/JPG/SVG | 0 | 0 | 0 | +| **Repo memory write** | ❌ | ✅ `memory/audit-workflows` branch | ❌ | ✅ `memory/meta-orchestrators` branch | ❌ | +| **Cache memory write** | ❌ | ❌ | ❌ | ❌ | ✅ `/tmp/gh-aw/cache-memory/safe-output-health/` | +| **noop** | ✅ (not reported as issue) | ❌ | ❌ | ❌ | ❌ | + +--- + +## Table 4 — Import Dependencies + +| Import | Kit | Audit | DOR | WHM | SOH | +|--------|-----|-------|-----|-----|-----| +| `shared/daily-audit-discussion.md` | ✅ (expires 7d) | ✅ (expires 1d) | ❌ (uses `daily-audit-base.md` instead) | ❌ | ✅ (expires 1d) | +| `shared/daily-audit-base.md` | ❌ | ❌ | ✅ | ❌ | ❌ | +| `shared/trending-charts-simple.md` | ✅ | ✅ | ❌ | ❌ | ❌ | +| `shared/reporting.md` | ✅ | ✅ | ❌ | ✅ | ✅ | +| `shared/repo-memory-standard.md` | ❌ | ✅ | ❌ | ❌ | ❌ | +| `shared/jqschema.md` | ❌ | ✅ | ❌ | ❌ | ✅ | +| `shared/aw-logs-24h-fetch.md` | ❌ | ❌ | ❌ | ❌ | ✅ | +| `.github/shared-instructions.md` (runtime import) | ❌ | ❌ | ✅ | ✅ | ❌ | + +--- + +## Table 5 — Overlap Map: KEEP vs. CONSOLIDATE + +Key: **✅ KEEP** = unique capability, no other workflow covers it. **⚠️ CONSOLIDATE** = overlaps with another; merge into the preference winner. + +| Capability / Signal | Kit | Audit | DOR | WHM | SOH | Preference Winner | +|--------------------|-----|-------|-----|-----|-----|-------------------| +| Daily "what broke in 24 h" run digest | ❌ | ✅ KEEP | ❌ | ⚠️ partial | ❌ | **Audit** | +| Weekly executive health narrative with escalation | ✅ KEEP | ❌ | ❌ | ❌ | ❌ | **Kit** | +| Episode/lineage DAG analysis | ✅ KEEP | ❌ | ❌ | ❌ | ❌ | **Kit** (only user) | +| Portfolio overlap + workflow value map | ✅ KEEP | ❌ | ❌ | ⚠️ stale detection | ❌ | **Kit** | +| 4-chart visual analytical suite | ✅ KEEP | ❌ | ❌ | ❌ | ❌ | **Kit** | +| 2-chart 30-day trend (health + cost) | ⚠️ redundant | ✅ KEEP | ❌ | ❌ | ❌ | **Audit** (charts match its daily cadence) | +| Firewall `access.log` coverage check | ❌ | ❌ | ✅ KEEP | ❌ | ❌ | **DOR** | +| MCP gateway `gateway.jsonl`/`rpc-messages.jsonl` check | ❌ | ❌ | ✅ KEEP | ❌ | ❌ | **DOR** | +| Infrastructure-layer telemetry (Squid, JSONL parsing) | ❌ | ❌ | ✅ KEEP | ❌ | ❌ | **DOR** | +| Workflow file + lock file structural health | ❌ | ❌ | ❌ | ✅ KEEP | ❌ | **WHM** | +| CI / Actions run health per workflow | ⚠️ partial | ⚠️ partial | ❌ | ✅ KEEP | ❌ | **WHM** | +| Repo-memory trend accumulation (30+ days) | ❌ | ✅ KEEP | ❌ | ✅ KEEP | ❌ | Both (different branches) | +| Safe-output job error clustering | ❌ | ❌ | ❌ | ⚠️ incidental | ✅ KEEP | **SOH** | +| Cache-memory persistent error patterns | ❌ | ❌ | ❌ | ❌ | ✅ KEEP | **SOH** | +| `[observability]` discussion prefix | ✅ | ❌ | ✅ **CONFLICT** | ❌ | ❌ | **Kit** (weekly); **DOR** should rename | +| Escalation issue creation | ✅ KEEP | ❌ | ❌ | ⚠️ overlap | ❌ | **Kit** for agentic escalation; **WHM** for structural | +| Missing-tool pattern detection | ❌ | ✅ KEEP | ❌ | ❌ | ❌ | **Audit** | +| MCP failure error analysis | ⚠️ risk signal | ✅ KEEP | ❌ | ❌ | ❌ | **Audit** (daily granularity is more actionable) | +| Token / cost reporting | ✅ (episode-level) | ✅ (daily trend) | ❌ | ❌ | ❌ | Both serve different cadences | +| Blocked-request reporting | ✅ (risk signal) | ✅ (error signal) | ✅ (coverage signal) | ❌ | ❌ | Distinct angles — all KEEP | + +--- + +## Table 6 — Key Conflicts Requiring Resolution + +| Conflict | Workflows | Impact | Recommended Resolution | +|----------|-----------|--------|------------------------| +| **Shared `[observability]` discussion title prefix** | Kit (expires 7d) + DOR (expires 1d, same prefix) | DOR's daily close-older-discussions call closes Kit's weekly report prematurely | Rename DOR prefix to `[firewall-mcp-coverage]` or `[telemetry-coverage]` | +| **Both analyze MCP failures** | Kit (risk score) + Audit (error log) | Dual-counting; readers get two different numbers for same failures | Separate concerns: Kit = trend/episode, Audit = raw daily errors | +| **Both escalate operational issues** | Kit (observability escalation issue) + WHM (cookie issues) | Two escalation channels, different labels, risk of duplication | Clarify: Kit escalates agentic behavior; WHM escalates structural/lock-file issues | +| **Both produce 30-day trend charts** | Kit (4 charts) + Audit (2 charts, same chart type: health + cost) | Workflow health and token/cost charts duplicated weekly vs. daily | Remove `trending-charts-simple` from Kit; Kit charts are already superior analytical replacements | +| **Both use `shared/trending-charts-simple.md`** | Kit + Audit | Shared import but Kit's charts supersede what `trending-charts-simple` adds | Drop `trending-charts-simple` import from Kit (use its own 4 custom charts) | +| **Audit has no `strict: true`** | Audit only | Audit can output to stdout without safe-output guard | Add `strict: true` to Audit if it should match Kit/SOH hygiene | + +--- + +## Table 7 — Engine Selection Rationale (Current State) + +| Workflow | Engine | Why (inferred) | Concerns | +|----------|--------|----------------|----------| +| Kit | copilot | Broad analytical reasoning over episode DAGs; portfolio judgments | Weekly cadence means higher per-run cost is acceptable | +| Audit | claude | Strong at structured log reading, pattern matching, natural language error summaries | Daily run; cost per run matters more than Kit | +| DOR | codex | Systematic file-existence checks; coverage math; tabular output | Least LLM-intensive task in this group; codex overhead may be overkill | +| WHM | copilot | Broad multi-step orchestration; file inspection + issue triage | `bash` and `edit` tools indicate it needs an agent that can act, not just report | +| SOH | claude | Error clustering and root-cause classification from log text | Daily run; structured output oriented | + +**Consolidation implication**: DOR is the strongest candidate for a model downgrade (codex → smaller/cheaper) since its task is primarily mechanical file-presence checking, not open-ended reasoning. + +--- + +## Table 8 — Summary: Five Workflows, Four Distinct Concerns + +| Concern | Primary Workflow | Secondary / Redundant | Action | +|---------|-----------------|----------------------|--------| +| **Agentic health narrative + portfolio** | Kit (weekly) | Audit (daily) partially overlaps | KEEP both; Kit is strategic, Audit is tactical | +| **Log artifact / telemetry coverage** | DOR | — | KEEP; rename discussion prefix | +| **Structural workflow file health** | WHM | — | KEEP; no overlapping workflow | +| **Safe-output job failures** | SOH | — | KEEP; no overlapping workflow | + +The pair with the most consolidation opportunity is **Kit ↔ Audit**. The four remaining workflows each own a distinct concern and have minimal overlap with each other. + +--- + +## Appendix: Raw Signal Inventory by Workflow + +### Kit — Unique Signals (not in Audit) +- `episodes[].episode_id`, `.kind`, `.confidence`, `.reasons[]` +- `episodes[].escalation_eligible`, `.escalation_reason`, `.suggested_route` +- `edges[].edge_type`, `.confidence` +- `behavior_fingerprint.*` (execution_style, tool_breadth, actuation_style, …) +- `comparison.baseline.selection`, `.matched_on[]` +- `comparison.classification.label`, `.reason_codes[]` +- `agentic_assessments[].kind`, `.severity` +- `workflow_instability_score`, `workflow_value_proxy`, `episode_risk_score` (derived) +- `workflow_overlap_score(a, b)` (pairwise, derived) +- Portfolio map chart (quadrant: keep/optimize/simplify/review) +- Overlap matrix chart (pairwise similarity heatmap) + +### Audit — Unique Signals (not in Kit) +- Per-run missing-tool pattern frequency +- Auth failure categorization +- `summary.engine_counts` from `logs` tool (engine classification) +- `repo-memory` pattern storage (`patterns/errors.json`, `patterns/missing-tools.json`) +- 24-hour granularity (Kit's minimum is ~7 days of meaningful signal) + +### DOR — Unique Signals (not in Kit or Audit) +- `access.log` file presence + Squid log entry count +- `gateway.jsonl` file presence + JSONL field validation +- `rpc-messages.jsonl` as canonical fallback telemetry source +- Firewall coverage percentage (`firewall_logs_present / firewall_enabled_workflows`) +- Gateway coverage percentage (`gateway_logs_present / mcp_enabled_workflows`) +- Per-run severity: CRITICAL / WARNING / HEALTHY / N/A classification + +### WHM — Unique Signals (not in any other workflow) +- Lock file existence for every `.md` file (structural completeness) +- Dependency graph between workflows (who imports whom) +- Health score per workflow (0–100, based on recent run success rate + error patterns) +- `memory/meta-orchestrators` shared repo memory +- Direct `edit` capability (can write fixes to workflow files) + +### SOH — Unique Signals (not in any other workflow) +- Safe-output job type breakdown (create_discussion, create_issue, add_comment, …) +- Error cluster analysis across safe-output job types +- Recurring failure tracking in cache-memory +- `error-patterns.json` + `recurring-failures.json` persistent state