diff --git a/.github/workflows/glossary-maintainer.lock.yml b/.github/workflows/glossary-maintainer.lock.yml index 6569f939d8..1f02547ddb 100644 --- a/.github/workflows/glossary-maintainer.lock.yml +++ b/.github/workflows/glossary-maintainer.lock.yml @@ -815,6 +815,152 @@ jobs: - Anticipate pitfalls and explain fixes empathetically. - Use alerts only when necessary. + ## Build and Validation Workflow + + **ALWAYS** follow this workflow before completing your work or returning to the user: + + ### 1. Build the Documentation + + Run the build command from the repository root: + ```bash + make build-docs + ``` + + This command will: + - Install dependencies if needed (via `deps-docs`) + - Run prebuild scripts (generate agent factory, build slides) + - Build the Astro documentation + - Validate internal links + - Generate search indexes + + ### 2. Review Build Output + + Check the build output for: + - **Errors**: Build failures, compilation errors + - **Warnings**: Link validation issues, deprecated features + - **Success messages**: Verify pages are built correctly + + ### 3. Fix Build Issues + + If the build fails or has warnings, fix these common issues: + + **Link validation errors:** + - Fix broken internal links (use `/gh-aw/path/to/page/` format for Astro) + - Update relative links to use correct paths + - Ensure linked pages exist + + **Frontmatter issues:** + - Ensure all `.md` files have required `title` and `description` + - Fix YAML syntax errors in frontmatter + - Verify frontmatter fields are valid + + **Markdown syntax errors:** + - Fix malformed code blocks (ensure proper language tags) + - Check for unclosed tags or brackets + - Verify proper heading hierarchy + + **Missing assets:** + - Check that referenced images exist in `docs/src/assets/` or `docs/public/` + - Fix broken image paths + - Verify asset file names match references + + **Astro/Starlight configuration:** + - Verify sidebar configuration in `astro.config.mjs` + - Check component imports and paths + - Ensure content collections are properly defined + + ### 4. Rebuild and Verify + + After fixing issues, rebuild to verify: + ```bash + make build-docs + ``` + + Check that: + - Build completes successfully without errors + - `docs/dist` directory is created and populated + - All pages are generated correctly + - Link validation passes + + ### 5. Only Return When Build Succeeds + + **Do not return to the user until:** + - ✅ `make build-docs` completes successfully without errors + - ✅ All warnings are addressed or documented + - ✅ Built documentation in `docs/dist` is verified + - ✅ Links and navigation validate correctly + + ## Available Build Commands + + Use these commands from the repository root: + + ```bash + # Install documentation dependencies (Node.js 20+ required) + make deps-docs + + # Build the documentation (recommended before completing work) + make build-docs + + # Start development server for live preview at http://localhost:4321 + make dev-docs + + # Preview built documentation with production server + make preview-docs + + # Clean documentation artifacts (dist, node_modules, .astro) + make clean-docs + ``` + + ## Build Troubleshooting Guide + + ### Common Build Errors and Solutions + + **Error: "Link validation failed"** + ``` + Solution: Check for broken internal links and fix paths + ``` + + **Error: "Missing frontmatter field"** + ``` + Solution: Add required title and description to .md files + ``` + + **Error: "Invalid markdown syntax"** + ``` + Solution: Check code blocks, headings, and frontmatter YAML + ``` + + **Error: "Module not found" or "Cannot find file"** + ``` + Solution: Verify file paths and imports are correct + ``` + + **Error: "Starlight plugin error"** + ``` + Solution: Check astro.config.mjs for configuration issues + ``` + + ### Debugging Process + + 1. **Read error messages carefully** - they usually indicate the exact issue + 2. **Check the failing file** - look at the file mentioned in the error + 3. **Fix the issue** - apply the appropriate solution + 4. **Rebuild** - run `make build-docs` again to verify + 5. **Repeat if needed** - continue until build succeeds + + ### Development Server for Testing + + Use the development server to preview changes in real-time: + ```bash + make dev-docs + ``` + + This starts Astro dev server at http://localhost:4321 with: + - Hot module reloading + - Fast refresh for instant updates + - Live error reporting + - Interactive debugging + ## Example Document Skeleton ```md --- @@ -871,6 +1017,29 @@ jobs: - Identify technical concepts and their relationships - Help generate clear, accurate definitions for technical terms - Understand how terms are used across the codebase + PROMPT_EOF + - name: Substitute placeholders + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + with: + script: | + const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE + } + }); + - name: Append prompt (part 2) + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + run: | + cat << 'PROMPT_EOF' >> "$GH_AW_PROMPT" ## Task Steps @@ -1002,29 +1171,6 @@ jobs: Definition of the term. Additional explanation if needed. Example: \`\`\`yaml - PROMPT_EOF - - name: Substitute placeholders - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - with: - script: | - const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); - - // Call the substitution function - return await substitutePlaceholders({ - file: process.env.GH_AW_PROMPT, - substitutions: { - GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE - } - }); - - name: Append prompt (part 2) - env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - run: | - cat << 'PROMPT_EOF' >> "$GH_AW_PROMPT" # Example code \`\`\` ``` diff --git a/pkg/parser/schemas/main_workflow_schema.json b/pkg/parser/schemas/main_workflow_schema.json index b15e5b3acd..6754a3d725 100644 --- a/pkg/parser/schemas/main_workflow_schema.json +++ b/pkg/parser/schemas/main_workflow_schema.json @@ -5944,6 +5944,10 @@ "type": "string", "description": "Custom user agent string for GitHub MCP server configuration (codex engine only)" }, + "command": { + "type": "string", + "description": "Custom executable path for the AI engine CLI. When specified, the workflow will skip the standard installation steps and use this command instead. The command should be the full path to the executable or a command available in PATH." + }, "env": { "type": "object", "description": "Custom environment variables to pass to the AI engine, including secret overrides (e.g., OPENAI_API_KEY: ${{ secrets.CUSTOM_KEY }})", diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index 43635c11fb..14706ad591 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -57,6 +57,12 @@ func (e *ClaudeEngine) GetRequiredSecretNames(workflowData *WorkflowData) []stri func (e *ClaudeEngine) GetInstallationSteps(workflowData *WorkflowData) []GitHubActionStep { claudeLog.Printf("Generating installation steps for Claude engine: workflow=%s", workflowData.Name) + // Skip installation if custom command is specified + if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { + claudeLog.Printf("Skipping installation steps: custom command specified (%s)", workflowData.EngineConfig.Command) + return []GitHubActionStep{} + } + var steps []GitHubActionStep // Define engine configuration for shared validation @@ -214,8 +220,17 @@ func (e *ClaudeEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str } // Build the command string with proper argument formatting - // Use claude command directly (available in PATH from hostedtoolcache mount) - commandParts := []string{"claude"} + // Determine which command to use + var commandName string + if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { + commandName = workflowData.EngineConfig.Command + claudeLog.Printf("Using custom command: %s", commandName) + } else { + // Use claude command directly (available in PATH from hostedtoolcache mount) + commandName = "claude" + } + + commandParts := []string{commandName} commandParts = append(commandParts, claudeArgs...) commandParts = append(commandParts, promptCommand) diff --git a/pkg/workflow/claude_engine_test.go b/pkg/workflow/claude_engine_test.go index ac840063b9..ead74777bb 100644 --- a/pkg/workflow/claude_engine_test.go +++ b/pkg/workflow/claude_engine_test.go @@ -511,3 +511,17 @@ func TestClaudeEngineNoDoubleEscapePrompt(t *testing.T) { } }) } + +func TestClaudeEngineSkipInstallationWithCommand(t *testing.T) { + engine := NewClaudeEngine() + + // Test with custom command - should skip installation + workflowData := &WorkflowData{ + EngineConfig: &EngineConfig{Command: "/usr/local/bin/custom-claude"}, + } + steps := engine.GetInstallationSteps(workflowData) + + if len(steps) != 0 { + t.Errorf("Expected 0 installation steps when command is specified, got %d", len(steps)) + } +} diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index 05003f8a91..4783d5524c 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -69,6 +69,12 @@ func (e *CodexEngine) GetRequiredSecretNames(workflowData *WorkflowData) []strin func (e *CodexEngine) GetInstallationSteps(workflowData *WorkflowData) []GitHubActionStep { codexEngineLog.Printf("Generating installation steps for Codex engine: workflow=%s", workflowData.Name) + // Skip installation if custom command is specified + if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { + codexEngineLog.Printf("Skipping installation steps: custom command specified (%s)", workflowData.EngineConfig.Command) + return []GitHubActionStep{} + } + // Use base installation steps (secret validation + npm install) steps := GetBaseInstallationSteps(EngineInstallConfig{ Secrets: []string{"CODEX_API_KEY", "OPENAI_API_KEY"}, @@ -158,10 +164,19 @@ func (e *CodexEngine) GetExecutionSteps(workflowData *WorkflowData, logFile stri } // Build the Codex command - // Use regular codex command regardless of firewall status - // PATH will be set to find codex in hostedtoolcache when firewall is enabled - codexCommand := fmt.Sprintf("codex %sexec%s%s%s\"$INSTRUCTION\"", - modelParam, webSearchParam, fullAutoParam, customArgsParam) + // Determine which command to use + var commandName string + if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { + commandName = workflowData.EngineConfig.Command + codexEngineLog.Printf("Using custom command: %s", commandName) + } else { + // Use regular codex command regardless of firewall status + // PATH will be set to find codex in hostedtoolcache when firewall is enabled + commandName = "codex" + } + + codexCommand := fmt.Sprintf("%s %sexec%s%s%s\"$INSTRUCTION\"", + commandName, modelParam, webSearchParam, fullAutoParam, customArgsParam) // Build the full command with agent file handling and AWF wrapping if enabled var command string @@ -289,18 +304,27 @@ mkdir -p "$CODEX_HOME/logs" } } else { // Build the command without AWF wrapping + // Determine which command to use + var commandName string + if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { + commandName = workflowData.EngineConfig.Command + codexEngineLog.Printf("Using custom command: %s", commandName) + } else { + commandName = "codex" + } + if workflowData.AgentFile != "" { agentPath := ResolveAgentFilePath(workflowData.AgentFile) command = fmt.Sprintf(`set -o pipefail AGENT_CONTENT="$(awk 'BEGIN{skip=1} /^---$/{if(skip){skip=0;next}else{skip=1;next}} !skip' %s)" INSTRUCTION="$(printf "%%s\n\n%%s" "$AGENT_CONTENT" "$(cat "$GH_AW_PROMPT")")" mkdir -p "$CODEX_HOME/logs" -codex %sexec%s%s%s"$INSTRUCTION" 2>&1 | tee %s`, agentPath, modelParam, webSearchParam, fullAutoParam, customArgsParam, logFile) +%s %sexec%s%s%s"$INSTRUCTION" 2>&1 | tee %s`, agentPath, commandName, modelParam, webSearchParam, fullAutoParam, customArgsParam, logFile) } else { command = fmt.Sprintf(`set -o pipefail INSTRUCTION="$(cat "$GH_AW_PROMPT")" mkdir -p "$CODEX_HOME/logs" -codex %sexec%s%s%s"$INSTRUCTION" 2>&1 | tee %s`, modelParam, webSearchParam, fullAutoParam, customArgsParam, logFile) +%s %sexec%s%s%s"$INSTRUCTION" 2>&1 | tee %s`, commandName, modelParam, webSearchParam, fullAutoParam, customArgsParam, logFile) } } diff --git a/pkg/workflow/codex_engine_test.go b/pkg/workflow/codex_engine_test.go index a65330798f..6ada3be67c 100644 --- a/pkg/workflow/codex_engine_test.go +++ b/pkg/workflow/codex_engine_test.go @@ -732,3 +732,17 @@ func TestCodexEngineHttpMCPServerRendered(t *testing.T) { }) } } + +func TestCodexEngineSkipInstallationWithCommand(t *testing.T) { + engine := NewCodexEngine() + + // Test with custom command - should skip installation + workflowData := &WorkflowData{ + EngineConfig: &EngineConfig{Command: "/usr/local/bin/custom-codex"}, + } + steps := engine.GetInstallationSteps(workflowData) + + if len(steps) != 0 { + t.Errorf("Expected 0 installation steps when command is specified, got %d", len(steps)) + } +} diff --git a/pkg/workflow/copilot_engine_execution.go b/pkg/workflow/copilot_engine_execution.go index 98d2602c97..b93d3988d2 100644 --- a/pkg/workflow/copilot_engine_execution.go +++ b/pkg/workflow/copilot_engine_execution.go @@ -151,18 +151,28 @@ func (e *CopilotEngine) GetExecutionSteps(workflowData *WorkflowData, logFile st if sandboxEnabled { // Build base command var baseCommand string - // For SRT: use locally installed package without -y flag to avoid internet fetch - // For AWF: use the installed binary directly - if isSRTEnabled(workflowData) { - // Use node explicitly to invoke copilot CLI to ensure env vars propagate correctly through sandbox - // The .bin/copilot shell wrapper doesn't properly pass environment variables through bubblewrap - // Environment variables are explicitly exported in the SRT wrapper to propagate through sandbox - baseCommand = fmt.Sprintf("node ./node_modules/.bin/copilot %s", shellJoinArgs(copilotArgs)) + + // Check if custom command is specified + var commandName string + if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { + commandName = workflowData.EngineConfig.Command + copilotExecLog.Printf("Using custom command: %s", commandName) } else { - // AWF - use the copilot binary installed by the installer script - // The binary is mounted into the AWF container from /usr/local/bin/copilot - baseCommand = fmt.Sprintf("/usr/local/bin/copilot %s", shellJoinArgs(copilotArgs)) + // For SRT: use locally installed package without -y flag to avoid internet fetch + // For AWF: use the installed binary directly + if isSRTEnabled(workflowData) { + // Use node explicitly to invoke copilot CLI to ensure env vars propagate correctly through sandbox + // The .bin/copilot shell wrapper doesn't properly pass environment variables through bubblewrap + // Environment variables are explicitly exported in the SRT wrapper to propagate through sandbox + commandName = "node ./node_modules/.bin/copilot" + } else { + // AWF - use the copilot binary installed by the installer script + // The binary is mounted into the AWF container from /usr/local/bin/copilot + commandName = "/usr/local/bin/copilot" + } } + + baseCommand = fmt.Sprintf("%s %s", commandName, shellJoinArgs(copilotArgs)) // Add conditional model flag if needed if needsModelFlag { @@ -171,8 +181,16 @@ func (e *CopilotEngine) GetExecutionSteps(workflowData *WorkflowData, logFile st copilotCommand = baseCommand } } else { - // When sandbox is disabled, use unpinned copilot command - baseCommand := fmt.Sprintf("copilot %s", shellJoinArgs(copilotArgs)) + // When sandbox is disabled, determine command to use + var commandName string + if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { + commandName = workflowData.EngineConfig.Command + copilotExecLog.Printf("Using custom command: %s", commandName) + } else { + commandName = "copilot" + } + + baseCommand := fmt.Sprintf("%s %s", commandName, shellJoinArgs(copilotArgs)) // Add conditional model flag if needed if needsModelFlag { diff --git a/pkg/workflow/copilot_engine_installation.go b/pkg/workflow/copilot_engine_installation.go index 8c17f57101..9b4bf360bc 100644 --- a/pkg/workflow/copilot_engine_installation.go +++ b/pkg/workflow/copilot_engine_installation.go @@ -31,9 +31,18 @@ var copilotInstallLog = logger.New("workflow:copilot_engine_installation") // 2. Node.js setup // 3. Sandbox installation (SRT or AWF, if needed) // 4. Copilot CLI installation +// +// If a custom command is specified in the engine configuration, this function returns +// an empty list of steps, skipping the standard installation process. func (e *CopilotEngine) GetInstallationSteps(workflowData *WorkflowData) []GitHubActionStep { copilotInstallLog.Printf("Generating installation steps for Copilot engine: workflow=%s", workflowData.Name) + // Skip installation if custom command is specified + if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { + copilotInstallLog.Printf("Skipping installation steps: custom command specified (%s)", workflowData.EngineConfig.Command) + return []GitHubActionStep{} + } + var steps []GitHubActionStep // Define engine configuration for shared validation diff --git a/pkg/workflow/copilot_engine_test.go b/pkg/workflow/copilot_engine_test.go index 41e64a15ad..66792c3123 100644 --- a/pkg/workflow/copilot_engine_test.go +++ b/pkg/workflow/copilot_engine_test.go @@ -1420,3 +1420,17 @@ func TestCopilotEngineParseLogMetrics_WithToolSizes(t *testing.T) { t.Errorf("Expected MaxOutputSize 0, got %d", githubTool.MaxOutputSize) } } + +func TestCopilotEngineSkipInstallationWithCommand(t *testing.T) { + engine := NewCopilotEngine() + + // Test with custom command - should skip installation + workflowData := &WorkflowData{ + EngineConfig: &EngineConfig{Command: "/usr/local/bin/custom-copilot"}, + } + steps := engine.GetInstallationSteps(workflowData) + + if len(steps) != 0 { + t.Errorf("Expected 0 installation steps when command is specified, got %d", len(steps)) + } +} diff --git a/pkg/workflow/engine.go b/pkg/workflow/engine.go index 10f782ac5c..efe2b19bc0 100644 --- a/pkg/workflow/engine.go +++ b/pkg/workflow/engine.go @@ -18,6 +18,7 @@ type EngineConfig struct { MaxTurns string Concurrency string // Agent job-level concurrency configuration (YAML format) UserAgent string + Command string // Custom executable path (when set, skip installation steps) Env map[string]string Steps []map[string]any Config string @@ -119,6 +120,13 @@ func (c *Compiler) ExtractEngineConfig(frontmatter map[string]any) (string, *Eng } } + // Extract optional 'command' field + if command, hasCommand := engineObj["command"]; hasCommand { + if commandStr, ok := command.(string); ok { + config.Command = commandStr + } + } + // Extract optional 'env' field (object/map of strings) if env, hasEnv := engineObj["env"]; hasEnv { if envMap, ok := env.(map[string]any); ok { diff --git a/pkg/workflow/engine_test.go b/pkg/workflow/engine_test.go index 357c702656..d8799138fa 100644 --- a/pkg/workflow/engine_test.go +++ b/pkg/workflow/engine_test.go @@ -183,3 +183,68 @@ func TestEngineVersionWithOtherFields(t *testing.T) { t.Errorf("Expected DEBUG='true', got %q", config.Env["DEBUG"]) } } + +// TestEngineCommandField tests that the command field is correctly extracted +func TestEngineCommandField(t *testing.T) { + tests := []struct { + name string + frontmatter map[string]any + expectedCommand string + }{ + { + name: "command field provided", + frontmatter: map[string]any{ + "engine": map[string]any{ + "id": "copilot", + "command": "/usr/local/bin/custom-copilot", + }, + }, + expectedCommand: "/usr/local/bin/custom-copilot", + }, + { + name: "command field not provided", + frontmatter: map[string]any{ + "engine": map[string]any{ + "id": "copilot", + }, + }, + expectedCommand: "", + }, + { + name: "command with relative path", + frontmatter: map[string]any{ + "engine": map[string]any{ + "id": "claude", + "command": "./bin/claude-cli", + }, + }, + expectedCommand: "./bin/claude-cli", + }, + { + name: "command with environment variable", + frontmatter: map[string]any{ + "engine": map[string]any{ + "id": "codex", + "command": "$HOME/.local/bin/codex", + }, + }, + expectedCommand: "$HOME/.local/bin/codex", + }, + } + + compiler := NewCompiler(false, "", "test") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, config := compiler.ExtractEngineConfig(tt.frontmatter) + + if config == nil { + t.Fatal("Expected config to be non-nil") + } + + if config.Command != tt.expectedCommand { + t.Errorf("Expected command %q, got %q", tt.expectedCommand, config.Command) + } + }) + } +}