From 2badd0cb4d54a23354e2b6482d5c0b54d5549150 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 25 Jan 2026 14:55:32 +0000 Subject: [PATCH 1/3] Initial plan From 83499ac23510e879df92fc483573d0c73463f898 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:11:10 +0000 Subject: [PATCH 2/3] Split compiler_orchestrator.go into 4 focused modules - Created compiler_orchestrator_frontmatter.go (167 lines) - Created compiler_orchestrator_engine.go (239 lines) - Created compiler_orchestrator_tools.go (258 lines) - Created compiler_orchestrator_workflow.go (450 lines) - Reduced original file to 9 lines (shared loggers only) All orchestrator tests pass. Build successful. Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/compiler_orchestrator.go | 1072 +---------------- pkg/workflow/compiler_orchestrator_engine.go | 239 ++++ .../compiler_orchestrator_frontmatter.go | 167 +++ pkg/workflow/compiler_orchestrator_tools.go | 258 ++++ .../compiler_orchestrator_workflow.go | 450 +++++++ 5 files changed, 1116 insertions(+), 1070 deletions(-) create mode 100644 pkg/workflow/compiler_orchestrator_engine.go create mode 100644 pkg/workflow/compiler_orchestrator_frontmatter.go create mode 100644 pkg/workflow/compiler_orchestrator_tools.go create mode 100644 pkg/workflow/compiler_orchestrator_workflow.go diff --git a/pkg/workflow/compiler_orchestrator.go b/pkg/workflow/compiler_orchestrator.go index a05d1c2b29..70232bf106 100644 --- a/pkg/workflow/compiler_orchestrator.go +++ b/pkg/workflow/compiler_orchestrator.go @@ -1,1077 +1,9 @@ package workflow import ( - "fmt" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/githubnext/gh-aw/pkg/console" - "github.com/githubnext/gh-aw/pkg/logger" - "github.com/githubnext/gh-aw/pkg/parser" - "github.com/goccy/go-yaml" +"github.com/githubnext/gh-aw/pkg/logger" ) +// Shared loggers used across compiler orchestrator modules var detectionLog = logger.New("workflow:detection") var orchestratorLog = logger.New("workflow:compiler_orchestrator") - -// frontmatterParseResult holds the results of parsing and validating frontmatter -type frontmatterParseResult struct { - cleanPath string - content []byte - frontmatterResult *parser.FrontmatterResult - frontmatterForValidation map[string]any - markdownDir string - isSharedWorkflow bool -} - -// engineSetupResult holds the results of engine configuration and validation -type engineSetupResult struct { - engineSetting string - engineConfig *EngineConfig - agenticEngine CodingAgentEngine - networkPermissions *NetworkPermissions - sandboxConfig *SandboxConfig - importsResult *parser.ImportsResult -} - -// toolsProcessingResult holds the results of tools and markdown processing -type toolsProcessingResult struct { - tools map[string]any - runtimes map[string]any - toolsTimeout int - toolsStartupTimeout int - markdownContent string - allIncludedFiles []string - workflowName string - frontmatterName string - needsTextOutput bool - trackerID string - safeOutputs *SafeOutputsConfig - secretMasking *SecretMaskingConfig - parsedFrontmatter *FrontmatterConfig -} - -// processToolsAndMarkdown processes tools configuration, runtimes, and markdown content. -// This function handles: -// - Safe outputs and secret masking configuration -// - Tools and MCP servers merging -// - Runtimes merging -// - MCP validations -// - Markdown content expansion -// - Workflow name extraction -func (c *Compiler) processToolsAndMarkdown(result *parser.FrontmatterResult, cleanPath string, markdownDir string, - agenticEngine CodingAgentEngine, engineSetting string, importsResult *parser.ImportsResult) (*toolsProcessingResult, error) { - - orchestratorLog.Printf("Processing tools and markdown") - log.Print("Processing tools and includes...") - - // Extract SafeOutputs configuration early so we can use it when applying default tools - safeOutputs := c.extractSafeOutputsConfig(result.Frontmatter) - - // Extract SecretMasking configuration - secretMasking := c.extractSecretMaskingConfig(result.Frontmatter) - - // Merge secret-masking from imports with top-level secret-masking - if importsResult.MergedSecretMasking != "" { - orchestratorLog.Printf("Merging secret-masking from imports") - var err error - secretMasking, err = c.MergeSecretMasking(secretMasking, importsResult.MergedSecretMasking) - if err != nil { - orchestratorLog.Printf("Secret-masking merge failed: %v", err) - return nil, fmt.Errorf("failed to merge secret-masking: %w", err) - } - } - - var tools map[string]any - - // Extract tools from the main file - topTools := extractToolsFromFrontmatter(result.Frontmatter) - - // Extract mcp-servers from the main file and merge them into tools - mcpServers := extractMCPServersFromFrontmatter(result.Frontmatter) - - // Process @include directives to extract additional tools - orchestratorLog.Printf("Expanding includes for tools") - includedTools, includedToolFiles, err := parser.ExpandIncludesWithManifest(result.Markdown, markdownDir, true) - if err != nil { - orchestratorLog.Printf("Failed to expand includes for tools: %v", err) - return nil, fmt.Errorf("failed to expand includes for tools: %w", err) - } - - // Combine imported tools with included tools - var allIncludedTools string - if importsResult.MergedTools != "" && includedTools != "" { - allIncludedTools = importsResult.MergedTools + "\n" + includedTools - } else if importsResult.MergedTools != "" { - allIncludedTools = importsResult.MergedTools - } else { - allIncludedTools = includedTools - } - - // Combine imported mcp-servers with top-level mcp-servers - // Imported mcp-servers are in JSON format (newline-separated), need to merge them - allMCPServers := mcpServers - if importsResult.MergedMCPServers != "" { - orchestratorLog.Printf("Merging imported mcp-servers") - // Parse and merge imported MCP servers - mergedMCPServers, err := c.MergeMCPServers(mcpServers, importsResult.MergedMCPServers) - if err != nil { - orchestratorLog.Printf("MCP servers merge failed: %v", err) - return nil, fmt.Errorf("failed to merge imported mcp-servers: %w", err) - } - allMCPServers = mergedMCPServers - } - - // Merge tools including mcp-servers - orchestratorLog.Printf("Merging tools and MCP servers") - tools, err = c.mergeToolsAndMCPServers(topTools, allMCPServers, allIncludedTools) - if err != nil { - orchestratorLog.Printf("Tools merge failed: %v", err) - return nil, fmt.Errorf("failed to merge tools: %w", err) - } - - // Extract and validate tools timeout settings - toolsTimeout, err := c.extractToolsTimeout(tools) - if err != nil { - return nil, fmt.Errorf("invalid tools timeout configuration: %w", err) - } - - toolsStartupTimeout, err := c.extractToolsStartupTimeout(tools) - if err != nil { - return nil, fmt.Errorf("invalid tools startup timeout configuration: %w", err) - } - - // Remove meta fields (timeout, startup-timeout) from merged tools map - // These are configuration fields, not actual tools - delete(tools, "timeout") - delete(tools, "startup-timeout") - - // Extract and merge runtimes from frontmatter and imports - topRuntimes := extractRuntimesFromFrontmatter(result.Frontmatter) - orchestratorLog.Printf("Merging runtimes") - runtimes, err := mergeRuntimes(topRuntimes, importsResult.MergedRuntimes) - if err != nil { - orchestratorLog.Printf("Runtimes merge failed: %v", err) - return nil, fmt.Errorf("failed to merge runtimes: %w", err) - } - - // Add MCP fetch server if needed (when web-fetch is requested but engine doesn't support it) - tools, _ = AddMCPFetchServerIfNeeded(tools, agenticEngine) - - // Validate MCP configurations - orchestratorLog.Printf("Validating MCP configurations") - if err := ValidateMCPConfigs(tools); err != nil { - orchestratorLog.Printf("MCP configuration validation failed: %v", err) - return nil, err - } - - // Validate HTTP transport support for the current engine - if err := c.validateHTTPTransportSupport(tools, agenticEngine); err != nil { - orchestratorLog.Printf("HTTP transport validation failed: %v", err) - return nil, err - } - - if !agenticEngine.SupportsToolsAllowlist() { - // For engines that don't support tool allowlists (like codex), ignore tools section and provide warnings - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Using experimental %s support (engine: %s)", agenticEngine.GetDisplayName(), engineSetting))) - c.IncrementWarningCount() - if _, hasTools := result.Frontmatter["tools"]; hasTools { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("'tools' section ignored when using engine: %s (%s doesn't support MCP tool allow-listing)", engineSetting, agenticEngine.GetDisplayName()))) - c.IncrementWarningCount() - } - tools = map[string]any{} - // For now, we'll add a basic github tool (always uses docker MCP) - githubConfig := map[string]any{} - tools["github"] = githubConfig - } - - // Validate max-turns support for the current engine - if err := c.validateMaxTurnsSupport(result.Frontmatter, agenticEngine); err != nil { - return nil, err - } - - // Validate web-search support for the current engine (warning only) - c.validateWebSearchSupport(tools, agenticEngine) - - // Process @include directives in markdown content - markdownContent, includedMarkdownFiles, err := parser.ExpandIncludesWithManifest(result.Markdown, markdownDir, false) - if err != nil { - return nil, fmt.Errorf("failed to expand includes in markdown: %w", err) - } - - // Prepend imported markdown from frontmatter imports field - if importsResult.MergedMarkdown != "" { - markdownContent = importsResult.MergedMarkdown + markdownContent - } - - log.Print("Expanded includes in markdown content") - - // Combine all included files (from tools and markdown) - // Use a map to deduplicate files - allIncludedFilesMap := make(map[string]bool) - for _, file := range includedToolFiles { - allIncludedFilesMap[file] = true - } - for _, file := range includedMarkdownFiles { - allIncludedFilesMap[file] = true - } - var allIncludedFiles []string - for file := range allIncludedFilesMap { - allIncludedFiles = append(allIncludedFiles, file) - } - // Sort files alphabetically to ensure consistent ordering in lock files - sort.Strings(allIncludedFiles) - - // Extract workflow name - workflowName, err := parser.ExtractWorkflowNameFromMarkdown(cleanPath) - if err != nil { - return nil, fmt.Errorf("failed to extract workflow name: %w", err) - } - - // Check if frontmatter specifies a custom name and use it instead - frontmatterName := extractStringFromMap(result.Frontmatter, "name", nil) - if frontmatterName != "" { - workflowName = frontmatterName - } - - log.Printf("Extracted workflow name: '%s'", workflowName) - - // Check if the markdown content uses the text output - needsTextOutput := c.detectTextOutputUsage(markdownContent) - - // Extract and validate tracker-id - trackerID, err := c.extractTrackerID(result.Frontmatter) - if err != nil { - return nil, err - } - - // Parse frontmatter config once for performance optimization - parsedFrontmatter, err := ParseFrontmatterConfig(result.Frontmatter) - if err != nil { - orchestratorLog.Printf("Failed to parse frontmatter config: %v", err) - // Non-fatal error - continue with nil ParsedFrontmatter - parsedFrontmatter = nil - } - - return &toolsProcessingResult{ - tools: tools, - runtimes: runtimes, - toolsTimeout: toolsTimeout, - toolsStartupTimeout: toolsStartupTimeout, - markdownContent: markdownContent, - allIncludedFiles: allIncludedFiles, - workflowName: workflowName, - frontmatterName: frontmatterName, - needsTextOutput: needsTextOutput, - trackerID: trackerID, - safeOutputs: safeOutputs, - secretMasking: secretMasking, - parsedFrontmatter: parsedFrontmatter, - }, nil -} - -// setupEngineAndImports configures the AI engine, processes imports, and validates network/sandbox settings. -// This function handles: -// - Engine extraction and validation -// - Import processing and merging -// - Network permissions setup -// - Sandbox configuration -// - Strict mode validations -func (c *Compiler) setupEngineAndImports(result *parser.FrontmatterResult, cleanPath string, content []byte, markdownDir string) (*engineSetupResult, error) { - orchestratorLog.Printf("Setting up engine and processing imports") - - // Extract AI engine setting from frontmatter - engineSetting, engineConfig := c.ExtractEngineConfig(result.Frontmatter) - - // Extract network permissions from frontmatter - networkPermissions := c.extractNetworkPermissions(result.Frontmatter) - - // Default to 'defaults' ecosystem if no network permissions specified - if networkPermissions == nil { - networkPermissions = &NetworkPermissions{ - Allowed: []string{"defaults"}, - } - } - - // Extract sandbox configuration from frontmatter - sandboxConfig := c.extractSandboxConfig(result.Frontmatter) - - // Save the initial strict mode state to restore it after this workflow is processed - // This ensures that strict mode from one workflow doesn't affect other workflows - initialStrictMode := c.strictMode - - // Check strict mode in frontmatter - // Priority: CLI flag > frontmatter > schema default (true) - if !c.strictMode { - // CLI flag not set, check frontmatter - if strictValue, exists := result.Frontmatter["strict"]; exists { - // Frontmatter explicitly sets strict mode - if strictBool, ok := strictValue.(bool); ok { - c.strictMode = strictBool - } - } else { - // Neither CLI nor frontmatter set - use schema default (true) - c.strictMode = true - } - } - - // Perform strict mode validations - orchestratorLog.Printf("Performing strict mode validation (strict=%v)", c.strictMode) - if err := c.validateStrictMode(result.Frontmatter, networkPermissions); err != nil { - orchestratorLog.Printf("Strict mode validation failed: %v", err) - // Restore strict mode before returning error - c.strictMode = initialStrictMode - return nil, err - } - - // Restore the initial strict mode state after validation - // This ensures strict mode doesn't leak to other workflows being compiled - c.strictMode = initialStrictMode - - // Override with command line AI engine setting if provided - if c.engineOverride != "" { - originalEngineSetting := engineSetting - if originalEngineSetting != "" && originalEngineSetting != c.engineOverride { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Command line --engine %s overrides markdown file engine: %s", c.engineOverride, originalEngineSetting))) - c.IncrementWarningCount() - } - engineSetting = c.engineOverride - } - - // Process imports from frontmatter first (before @include directives) - orchestratorLog.Printf("Processing imports from frontmatter") - importCache := c.getSharedImportCache() - // Pass the full file content for accurate line/column error reporting - importsResult, err := parser.ProcessImportsFromFrontmatterWithSource(result.Frontmatter, markdownDir, importCache, cleanPath, string(content)) - if err != nil { - orchestratorLog.Printf("Import processing failed: %v", err) - return nil, err // Error is already formatted with source location - } - - // Merge network permissions from imports with top-level network permissions - if importsResult.MergedNetwork != "" { - orchestratorLog.Printf("Merging network permissions from imports") - networkPermissions, err = c.MergeNetworkPermissions(networkPermissions, importsResult.MergedNetwork) - if err != nil { - orchestratorLog.Printf("Network permissions merge failed: %v", err) - return nil, fmt.Errorf("failed to merge network permissions: %w", err) - } - } - - // Validate permissions from imports against top-level permissions - // Extract top-level permissions first - topLevelPermissions := c.extractPermissions(result.Frontmatter) - if importsResult.MergedPermissions != "" { - orchestratorLog.Printf("Validating included permissions") - if err := c.ValidateIncludedPermissions(topLevelPermissions, importsResult.MergedPermissions); err != nil { - orchestratorLog.Printf("Included permissions validation failed: %v", err) - return nil, fmt.Errorf("permission validation failed: %w", err) - } - } - - // Process @include directives to extract engine configurations and check for conflicts - orchestratorLog.Printf("Expanding includes for engine configurations") - includedEngines, err := parser.ExpandIncludesForEngines(result.Markdown, markdownDir) - if err != nil { - orchestratorLog.Printf("Failed to expand includes for engines: %v", err) - return nil, fmt.Errorf("failed to expand includes for engines: %w", err) - } - - // Combine imported engines with included engines - allEngines := append(importsResult.MergedEngines, includedEngines...) - - // Validate that only one engine field exists across all files - orchestratorLog.Printf("Validating single engine specification") - finalEngineSetting, err := c.validateSingleEngineSpecification(engineSetting, allEngines) - if err != nil { - orchestratorLog.Printf("Engine specification validation failed: %v", err) - return nil, err - } - if finalEngineSetting != "" { - engineSetting = finalEngineSetting - } - - // If engineConfig is nil (engine was in an included file), extract it from the included engine JSON - if engineConfig == nil && len(allEngines) > 0 { - orchestratorLog.Printf("Extracting engine config from included file") - extractedConfig, err := c.extractEngineConfigFromJSON(allEngines[0]) - if err != nil { - orchestratorLog.Printf("Failed to extract engine config: %v", err) - return nil, fmt.Errorf("failed to extract engine config from included file: %w", err) - } - engineConfig = extractedConfig - } - - // Apply the default AI engine setting if not specified - if engineSetting == "" { - defaultEngine := c.engineRegistry.GetDefaultEngine() - engineSetting = defaultEngine.GetID() - log.Printf("No 'engine:' setting found, defaulting to: %s", engineSetting) - // Create a default EngineConfig with the default engine ID if not already set - if engineConfig == nil { - engineConfig = &EngineConfig{ID: engineSetting} - } else if engineConfig.ID == "" { - engineConfig.ID = engineSetting - } - } - - // Validate the engine setting - orchestratorLog.Printf("Validating engine setting: %s", engineSetting) - if err := c.validateEngine(engineSetting); err != nil { - orchestratorLog.Printf("Engine validation failed: %v", err) - return nil, err - } - - // Get the agentic engine instance - agenticEngine, err := c.getAgenticEngine(engineSetting) - if err != nil { - orchestratorLog.Printf("Failed to get agentic engine: %v", err) - return nil, err - } - - log.Printf("AI engine: %s (%s)", agenticEngine.GetDisplayName(), engineSetting) - if agenticEngine.IsExperimental() && c.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Using experimental engine: %s", agenticEngine.GetDisplayName()))) - c.IncrementWarningCount() - } - - // Enable firewall by default for copilot engine when network restrictions are present - // (unless SRT sandbox is configured, since AWF and SRT are mutually exclusive) - enableFirewallByDefaultForCopilot(engineSetting, networkPermissions, sandboxConfig) - - // Enable firewall by default for claude engine when network restrictions are present - enableFirewallByDefaultForClaude(engineSetting, networkPermissions, sandboxConfig) - - // Re-evaluate strict mode for firewall and network validation - // (it was restored after validateStrictMode but we need it again) - initialStrictModeForFirewall := c.strictMode - if !c.strictMode { - // CLI flag not set, check frontmatter - if strictValue, exists := result.Frontmatter["strict"]; exists { - // Frontmatter explicitly sets strict mode - if strictBool, ok := strictValue.(bool); ok { - c.strictMode = strictBool - } - } else { - // Neither CLI nor frontmatter set - use schema default (true) - c.strictMode = true - } - } - - // Validate firewall is enabled in strict mode for copilot with network restrictions - orchestratorLog.Printf("Validating strict firewall (strict=%v)", c.strictMode) - if err := c.validateStrictFirewall(engineSetting, networkPermissions, sandboxConfig); err != nil { - orchestratorLog.Printf("Strict firewall validation failed: %v", err) - c.strictMode = initialStrictModeForFirewall - return nil, err - } - - // Check if the engine supports network restrictions when they are defined - if err := c.checkNetworkSupport(agenticEngine, networkPermissions); err != nil { - orchestratorLog.Printf("Network support check failed: %v", err) - // Restore strict mode before returning error - c.strictMode = initialStrictModeForFirewall - return nil, err - } - - // Restore the strict mode state after network check - c.strictMode = initialStrictModeForFirewall - - return &engineSetupResult{ - engineSetting: engineSetting, - engineConfig: engineConfig, - agenticEngine: agenticEngine, - networkPermissions: networkPermissions, - sandboxConfig: sandboxConfig, - importsResult: importsResult, - }, nil -} - -// parseFrontmatterSection reads the workflow file and parses its frontmatter. -// It returns a frontmatterParseResult containing the parsed data and validation information. -// If the workflow is detected as a shared workflow (no 'on' field), isSharedWorkflow is set to true. -func (c *Compiler) parseFrontmatterSection(markdownPath string) (*frontmatterParseResult, error) { - orchestratorLog.Printf("Starting frontmatter parsing: %s", markdownPath) - log.Printf("Reading file: %s", markdownPath) - - // Clean the path to prevent path traversal issues (gosec G304) - // filepath.Clean removes ".." and other problematic path elements - cleanPath := filepath.Clean(markdownPath) - - // Read the file - content, err := os.ReadFile(cleanPath) - if err != nil { - orchestratorLog.Printf("Failed to read file: %s, error: %v", cleanPath, err) - return nil, fmt.Errorf("failed to read file: %w", err) - } - - log.Printf("File size: %d bytes", len(content)) - - // Parse frontmatter and markdown - orchestratorLog.Printf("Parsing frontmatter from file: %s", cleanPath) - result, err := parser.ExtractFrontmatterFromContent(string(content)) - if err != nil { - orchestratorLog.Printf("Frontmatter extraction failed: %v", err) - // Use FrontmatterStart from result if available, otherwise default to line 2 (after opening ---) - frontmatterStart := 2 - if result != nil && result.FrontmatterStart > 0 { - frontmatterStart = result.FrontmatterStart - } - return nil, c.createFrontmatterError(cleanPath, string(content), err, frontmatterStart) - } - - if len(result.Frontmatter) == 0 { - orchestratorLog.Print("No frontmatter found in file") - return nil, fmt.Errorf("no frontmatter found") - } - - // Preprocess schedule fields to convert human-friendly format to cron expressions - if err := c.preprocessScheduleFields(result.Frontmatter, cleanPath, string(content)); err != nil { - orchestratorLog.Printf("Schedule preprocessing failed: %v", err) - return nil, err - } - - // Create a copy of frontmatter without internal markers for schema validation - // Keep the original frontmatter with markers for YAML generation - frontmatterForValidation := c.copyFrontmatterWithoutInternalMarkers(result.Frontmatter) - - // Check if "on" field is missing - if so, treat as a shared/imported workflow - _, hasOnField := frontmatterForValidation["on"] - if !hasOnField { - detectionLog.Printf("No 'on' field detected - treating as shared agentic workflow") - - // Validate as an included/shared workflow (uses main_workflow_schema with forbidden field checks) - if err := parser.ValidateIncludedFileFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { - orchestratorLog.Printf("Shared workflow validation failed: %v", err) - return nil, err - } - - return &frontmatterParseResult{ - cleanPath: cleanPath, - content: content, - frontmatterResult: result, - frontmatterForValidation: frontmatterForValidation, - markdownDir: filepath.Dir(cleanPath), - isSharedWorkflow: true, - }, nil - } - - // For main workflows (with 'on' field), markdown content is required - if result.Markdown == "" { - orchestratorLog.Print("No markdown content found for main workflow") - return nil, fmt.Errorf("no markdown content found") - } - - // Validate main workflow frontmatter contains only expected entries - orchestratorLog.Printf("Validating main workflow frontmatter schema") - if err := parser.ValidateMainWorkflowFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { - orchestratorLog.Printf("Main workflow frontmatter validation failed: %v", err) - return nil, err - } - - // Validate event filter mutual exclusivity (branches/branches-ignore, paths/paths-ignore) - if err := ValidateEventFilters(frontmatterForValidation); err != nil { - orchestratorLog.Printf("Event filter validation failed: %v", err) - return nil, err - } - - // Validate that @include/@import directives are not used inside template regions - if err := validateNoIncludesInTemplateRegions(result.Markdown); err != nil { - orchestratorLog.Printf("Template region validation failed: %v", err) - return nil, fmt.Errorf("template region validation failed: %w", err) - } - - log.Printf("Frontmatter: %d chars, Markdown: %d chars", len(result.Frontmatter), len(result.Markdown)) - - return &frontmatterParseResult{ - cleanPath: cleanPath, - content: content, - frontmatterResult: result, - frontmatterForValidation: frontmatterForValidation, - markdownDir: filepath.Dir(cleanPath), - isSharedWorkflow: false, - }, nil -} - -// buildInitialWorkflowData creates the initial WorkflowData struct with basic fields populated -func (c *Compiler) buildInitialWorkflowData( - result *parser.FrontmatterResult, - toolsResult *toolsProcessingResult, - engineSetup *engineSetupResult, - importsResult *parser.ImportsResult, -) *WorkflowData { - orchestratorLog.Print("Building initial workflow data") - - return &WorkflowData{ - Name: toolsResult.workflowName, - FrontmatterName: toolsResult.frontmatterName, - FrontmatterYAML: strings.Join(result.FrontmatterLines, "\n"), - Description: c.extractDescription(result.Frontmatter), - Source: c.extractSource(result.Frontmatter), - TrackerID: toolsResult.trackerID, - ImportedFiles: importsResult.ImportedFiles, - IncludedFiles: toolsResult.allIncludedFiles, - ImportInputs: importsResult.ImportInputs, - Tools: toolsResult.tools, - ParsedTools: NewTools(toolsResult.tools), - Runtimes: toolsResult.runtimes, - MarkdownContent: toolsResult.markdownContent, - AI: engineSetup.engineSetting, - EngineConfig: engineSetup.engineConfig, - AgentFile: importsResult.AgentFile, - NetworkPermissions: engineSetup.networkPermissions, - SandboxConfig: applySandboxDefaults(engineSetup.sandboxConfig, engineSetup.engineConfig), - NeedsTextOutput: toolsResult.needsTextOutput, - ToolsTimeout: toolsResult.toolsTimeout, - ToolsStartupTimeout: toolsResult.toolsStartupTimeout, - TrialMode: c.trialMode, - TrialLogicalRepo: c.trialLogicalRepoSlug, - GitHubToken: extractStringFromMap(result.Frontmatter, "github-token", nil), - StrictMode: c.strictMode, - SecretMasking: toolsResult.secretMasking, - ParsedFrontmatter: toolsResult.parsedFrontmatter, - } -} - -// extractYAMLSections extracts YAML configuration sections from frontmatter -func (c *Compiler) extractYAMLSections(frontmatter map[string]any, workflowData *WorkflowData) { - orchestratorLog.Print("Extracting YAML sections from frontmatter") - - workflowData.On = c.extractTopLevelYAMLSection(frontmatter, "on") - workflowData.Permissions = c.extractPermissions(frontmatter) - workflowData.Network = c.extractTopLevelYAMLSection(frontmatter, "network") - workflowData.Concurrency = c.extractTopLevelYAMLSection(frontmatter, "concurrency") - workflowData.RunName = c.extractTopLevelYAMLSection(frontmatter, "run-name") - workflowData.Env = c.extractTopLevelYAMLSection(frontmatter, "env") - workflowData.Features = c.extractFeatures(frontmatter) - workflowData.If = c.extractIfCondition(frontmatter) - - // Prefer timeout-minutes (new) over timeout_minutes (deprecated) - workflowData.TimeoutMinutes = c.extractTopLevelYAMLSection(frontmatter, "timeout-minutes") - if workflowData.TimeoutMinutes == "" { - workflowData.TimeoutMinutes = c.extractTopLevelYAMLSection(frontmatter, "timeout_minutes") - if workflowData.TimeoutMinutes != "" { - // Emit deprecation warning - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Field 'timeout_minutes' is deprecated. Please use 'timeout-minutes' instead to follow GitHub Actions naming convention.")) - } - } - - workflowData.RunsOn = c.extractTopLevelYAMLSection(frontmatter, "runs-on") - workflowData.Environment = c.extractTopLevelYAMLSection(frontmatter, "environment") - workflowData.Container = c.extractTopLevelYAMLSection(frontmatter, "container") - workflowData.Cache = c.extractTopLevelYAMLSection(frontmatter, "cache") -} - -// processAndMergeSteps handles the merging of imported steps with main workflow steps -func (c *Compiler) processAndMergeSteps(frontmatter map[string]any, workflowData *WorkflowData, importsResult *parser.ImportsResult) { - orchestratorLog.Print("Processing and merging custom steps") - - workflowData.CustomSteps = c.extractTopLevelYAMLSection(frontmatter, "steps") - - // Merge imported steps if any - if importsResult.MergedSteps != "" { - // Parse imported steps from YAML array - var importedSteps []any - if err := yaml.Unmarshal([]byte(importsResult.MergedSteps), &importedSteps); err == nil { - // Apply action pinning to imported steps - importedSteps = ApplyActionPinsToSteps(importedSteps, workflowData) - - // If there are main workflow steps, parse and merge them - if workflowData.CustomSteps != "" { - // Parse main workflow steps (format: "steps:\n - ...") - var mainStepsWrapper map[string]any - if err := yaml.Unmarshal([]byte(workflowData.CustomSteps), &mainStepsWrapper); err == nil { - if mainStepsVal, hasSteps := mainStepsWrapper["steps"]; hasSteps { - if mainSteps, ok := mainStepsVal.([]any); ok { - // Apply action pinning to main steps - mainSteps = ApplyActionPinsToSteps(mainSteps, workflowData) - - // Prepend imported steps to main steps - allSteps := append(importedSteps, mainSteps...) - // Convert back to YAML with "steps:" wrapper - stepsWrapper := map[string]any{"steps": allSteps} - stepsYAML, err := yaml.Marshal(stepsWrapper) - if err == nil { - // Remove quotes from uses values with version comments - workflowData.CustomSteps = unquoteUsesWithComments(string(stepsYAML)) - } - } - } - } - } else { - // Only imported steps exist, wrap in "steps:" format - stepsWrapper := map[string]any{"steps": importedSteps} - stepsYAML, err := yaml.Marshal(stepsWrapper) - if err == nil { - // Remove quotes from uses values with version comments - workflowData.CustomSteps = unquoteUsesWithComments(string(stepsYAML)) - } - } - } - } else if workflowData.CustomSteps != "" { - // No imported steps, but there are main steps - still apply pinning - var mainStepsWrapper map[string]any - if err := yaml.Unmarshal([]byte(workflowData.CustomSteps), &mainStepsWrapper); err == nil { - if mainStepsVal, hasSteps := mainStepsWrapper["steps"]; hasSteps { - if mainSteps, ok := mainStepsVal.([]any); ok { - // Apply action pinning to main steps - mainSteps = ApplyActionPinsToSteps(mainSteps, workflowData) - - // Convert back to YAML with "steps:" wrapper - stepsWrapper := map[string]any{"steps": mainSteps} - stepsYAML, err := yaml.Marshal(stepsWrapper) - if err == nil { - // Remove quotes from uses values with version comments - workflowData.CustomSteps = unquoteUsesWithComments(string(stepsYAML)) - } - } - } - } - } -} - -// processAndMergePostSteps handles the processing of post-steps with action pinning -func (c *Compiler) processAndMergePostSteps(frontmatter map[string]any, workflowData *WorkflowData) { - orchestratorLog.Print("Processing post-steps") - - workflowData.PostSteps = c.extractTopLevelYAMLSection(frontmatter, "post-steps") - - // Apply action pinning to post-steps if any - if workflowData.PostSteps != "" { - var postStepsWrapper map[string]any - if err := yaml.Unmarshal([]byte(workflowData.PostSteps), &postStepsWrapper); err == nil { - if postStepsVal, hasPostSteps := postStepsWrapper["post-steps"]; hasPostSteps { - if postSteps, ok := postStepsVal.([]any); ok { - // Apply action pinning to post steps - postSteps = ApplyActionPinsToSteps(postSteps, workflowData) - - // Convert back to YAML with "post-steps:" wrapper - stepsWrapper := map[string]any{"post-steps": postSteps} - stepsYAML, err := yaml.Marshal(stepsWrapper) - if err == nil { - // Remove quotes from uses values with version comments - workflowData.PostSteps = unquoteUsesWithComments(string(stepsYAML)) - } - } - } - } - } -} - -// processAndMergeServices handles the merging of imported services with main workflow services -func (c *Compiler) processAndMergeServices(frontmatter map[string]any, workflowData *WorkflowData, importsResult *parser.ImportsResult) { - orchestratorLog.Print("Processing and merging services") - - workflowData.Services = c.extractTopLevelYAMLSection(frontmatter, "services") - - // Merge imported services if any - if importsResult.MergedServices != "" { - // Parse imported services from YAML - var importedServices map[string]any - if err := yaml.Unmarshal([]byte(importsResult.MergedServices), &importedServices); err == nil { - // If there are main workflow services, parse and merge them - if workflowData.Services != "" { - // Parse main workflow services - var mainServicesWrapper map[string]any - if err := yaml.Unmarshal([]byte(workflowData.Services), &mainServicesWrapper); err == nil { - if mainServices, ok := mainServicesWrapper["services"].(map[string]any); ok { - // Merge: main workflow services take precedence over imported - for key, value := range importedServices { - if _, exists := mainServices[key]; !exists { - mainServices[key] = value - } - } - // Convert back to YAML with "services:" wrapper - servicesWrapper := map[string]any{"services": mainServices} - servicesYAML, err := yaml.Marshal(servicesWrapper) - if err == nil { - workflowData.Services = string(servicesYAML) - } - } - } - } else { - // Only imported services exist, wrap in "services:" format - servicesWrapper := map[string]any{"services": importedServices} - servicesYAML, err := yaml.Marshal(servicesWrapper) - if err == nil { - workflowData.Services = string(servicesYAML) - } - } - } - } -} - -// extractAdditionalConfigurations extracts cache-memory, repo-memory, safe-inputs, and safe-outputs configurations -func (c *Compiler) extractAdditionalConfigurations( - frontmatter map[string]any, - tools map[string]any, - markdownDir string, - workflowData *WorkflowData, - importsResult *parser.ImportsResult, - markdown string, - safeOutputs *SafeOutputsConfig, -) error { - orchestratorLog.Print("Extracting additional configurations") - - // Extract cache-memory config and check for errors - cacheMemoryConfig, err := c.extractCacheMemoryConfigFromMap(tools) - if err != nil { - return err - } - workflowData.CacheMemoryConfig = cacheMemoryConfig - - // Extract repo-memory config and check for errors - toolsConfig, err := ParseToolsConfig(tools) - if err != nil { - return err - } - repoMemoryConfig, err := c.extractRepoMemoryConfig(toolsConfig) - if err != nil { - return err - } - workflowData.RepoMemoryConfig = repoMemoryConfig - - // Extract and process safe-inputs and safe-outputs - workflowData.Command, workflowData.CommandEvents = c.extractCommandConfig(frontmatter) - workflowData.Jobs = c.extractJobsFromFrontmatter(frontmatter) - workflowData.Roles = c.extractRoles(frontmatter) - workflowData.Bots = c.extractBots(frontmatter) - - // Use the already extracted output configuration - workflowData.SafeOutputs = safeOutputs - - // Extract safe-inputs configuration - workflowData.SafeInputs = c.extractSafeInputsConfig(frontmatter) - - // Merge safe-inputs from imports - if len(importsResult.MergedSafeInputs) > 0 { - workflowData.SafeInputs = c.mergeSafeInputs(workflowData.SafeInputs, importsResult.MergedSafeInputs) - } - - // Extract safe-jobs from safe-outputs.jobs location - topSafeJobs := extractSafeJobsFromFrontmatter(frontmatter) - - // Process @include directives to extract additional safe-outputs configurations - includedSafeOutputsConfigs, err := parser.ExpandIncludesForSafeOutputs(markdown, markdownDir) - if err != nil { - return fmt.Errorf("failed to expand includes for safe-outputs: %w", err) - } - - // Combine imported safe-outputs with included safe-outputs - var allSafeOutputsConfigs []string - if len(importsResult.MergedSafeOutputs) > 0 { - allSafeOutputsConfigs = append(allSafeOutputsConfigs, importsResult.MergedSafeOutputs...) - } - if len(includedSafeOutputsConfigs) > 0 { - allSafeOutputsConfigs = append(allSafeOutputsConfigs, includedSafeOutputsConfigs...) - } - - // Merge safe-jobs from all safe-outputs configurations (imported and included) - includedSafeJobs, err := c.mergeSafeJobsFromIncludedConfigs(topSafeJobs, allSafeOutputsConfigs) - if err != nil { - return fmt.Errorf("failed to merge safe-jobs from includes: %w", err) - } - - // Merge app configuration from included safe-outputs configurations - includedApp, err := c.mergeAppFromIncludedConfigs(workflowData.SafeOutputs, allSafeOutputsConfigs) - if err != nil { - return fmt.Errorf("failed to merge app from includes: %w", err) - } - - // Ensure SafeOutputs exists and populate the Jobs field with merged jobs - if workflowData.SafeOutputs == nil && len(includedSafeJobs) > 0 { - workflowData.SafeOutputs = &SafeOutputsConfig{} - } - // Always use the merged includedSafeJobs as it contains both main and imported jobs - if workflowData.SafeOutputs != nil && len(includedSafeJobs) > 0 { - workflowData.SafeOutputs.Jobs = includedSafeJobs - } - - // Populate the App field if it's not set in the top-level workflow but is in an included config - if workflowData.SafeOutputs != nil && workflowData.SafeOutputs.App == nil && includedApp != nil { - workflowData.SafeOutputs.App = includedApp - } - - // Merge safe-outputs types from imports - mergedSafeOutputs, err := c.MergeSafeOutputs(workflowData.SafeOutputs, allSafeOutputsConfigs) - if err != nil { - return fmt.Errorf("failed to merge safe-outputs from imports: %w", err) - } - workflowData.SafeOutputs = mergedSafeOutputs - - return nil -} - -// processOnSectionAndFilters processes the on section configuration and applies various filters -func (c *Compiler) processOnSectionAndFilters( - frontmatter map[string]any, - workflowData *WorkflowData, - cleanPath string, -) error { - orchestratorLog.Print("Processing on section and filters") - - // Process stop-after configuration from the on: section - if err := c.processStopAfterConfiguration(frontmatter, workflowData, cleanPath); err != nil { - return err - } - - // Process skip-if-match configuration from the on: section - if err := c.processSkipIfMatchConfiguration(frontmatter, workflowData); err != nil { - return err - } - - // Process skip-if-no-match configuration from the on: section - if err := c.processSkipIfNoMatchConfiguration(frontmatter, workflowData); err != nil { - return err - } - - // Process manual-approval configuration from the on: section - if err := c.processManualApprovalConfiguration(frontmatter, workflowData); err != nil { - return err - } - - // Parse the "on" section for command triggers, reactions, and other events - if err := c.parseOnSection(frontmatter, workflowData, cleanPath); err != nil { - return err - } - - // Apply defaults - if err := c.applyDefaults(workflowData, cleanPath); err != nil { - return err - } - - // Apply pull request draft filter if specified - c.applyPullRequestDraftFilter(workflowData, frontmatter) - - // Apply pull request fork filter if specified - c.applyPullRequestForkFilter(workflowData, frontmatter) - - // Apply label filter if specified - c.applyLabelFilter(workflowData, frontmatter) - - return nil -} - -func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) { - orchestratorLog.Printf("Starting workflow file parsing: %s", markdownPath) - - // Parse frontmatter section - parseResult, err := c.parseFrontmatterSection(markdownPath) - if err != nil { - return nil, err - } - - // Handle shared workflows - if parseResult.isSharedWorkflow { - return nil, &SharedWorkflowError{Path: parseResult.cleanPath} - } - - // Unpack parse result for convenience - cleanPath := parseResult.cleanPath - content := parseResult.content - result := parseResult.frontmatterResult - markdownDir := parseResult.markdownDir - - // Setup engine and process imports - engineSetup, err := c.setupEngineAndImports(result, cleanPath, content, markdownDir) - if err != nil { - return nil, err - } - - // Process tools and markdown - toolsResult, err := c.processToolsAndMarkdown(result, cleanPath, markdownDir, engineSetup.agenticEngine, engineSetup.engineSetting, engineSetup.importsResult) - if err != nil { - return nil, err - } - - // Build initial workflow data structure - workflowData := c.buildInitialWorkflowData(result, toolsResult, engineSetup, engineSetup.importsResult) - - // Use shared action cache and resolver from the compiler - actionCache, actionResolver := c.getSharedActionResolver() - workflowData.ActionCache = actionCache - workflowData.ActionResolver = actionResolver - workflowData.ActionPinWarnings = c.actionPinWarnings - - // Extract YAML configuration sections from frontmatter - c.extractYAMLSections(result.Frontmatter, workflowData) - - // Process and merge custom steps with imported steps - c.processAndMergeSteps(result.Frontmatter, workflowData, engineSetup.importsResult) - - // Process and merge post-steps - c.processAndMergePostSteps(result.Frontmatter, workflowData) - - // Process and merge services - c.processAndMergeServices(result.Frontmatter, workflowData, engineSetup.importsResult) - - // Extract additional configurations (cache, safe-inputs, safe-outputs, etc.) - if err := c.extractAdditionalConfigurations( - result.Frontmatter, - toolsResult.tools, - markdownDir, - workflowData, - engineSetup.importsResult, - result.Markdown, - toolsResult.safeOutputs, - ); err != nil { - return nil, err - } - - // Process on section configuration and apply filters - if err := c.processOnSectionAndFilters(result.Frontmatter, workflowData, cleanPath); err != nil { - return nil, err - } - - orchestratorLog.Printf("Workflow file parsing completed successfully: %s", markdownPath) - return workflowData, nil -} - -// copyFrontmatterWithoutInternalMarkers creates a deep copy of frontmatter without internal marker fields -// This is used for schema validation while preserving markers in the original for YAML generation -func (c *Compiler) copyFrontmatterWithoutInternalMarkers(frontmatter map[string]any) map[string]any { - // Create a shallow copy of the top level - copy := make(map[string]any) - for k, v := range frontmatter { - if k == "on" { - // Special handling for "on" field - need to deep copy and remove markers - if onMap, ok := v.(map[string]any); ok { - onCopy := make(map[string]any) - for onKey, onValue := range onMap { - if onKey == "issues" || onKey == "pull_request" || onKey == "discussion" { - // Deep copy the section and remove marker - if sectionMap, ok := onValue.(map[string]any); ok { - sectionCopy := make(map[string]any) - for sectionKey, sectionValue := range sectionMap { - if sectionKey != "__gh_aw_native_label_filter__" { - sectionCopy[sectionKey] = sectionValue - } - } - onCopy[onKey] = sectionCopy - } else { - onCopy[onKey] = onValue - } - } else { - onCopy[onKey] = onValue - } - } - copy[k] = onCopy - } else { - copy[k] = v - } - } else { - copy[k] = v - } - } - return copy -} - -// detectTextOutputUsage checks if the markdown content uses ${{ needs.activation.outputs.text }} -func (c *Compiler) detectTextOutputUsage(markdownContent string) bool { - // Check for the specific GitHub Actions expression - hasUsage := strings.Contains(markdownContent, "${{ needs.activation.outputs.text }}") - detectionLog.Printf("Detected usage of activation.outputs.text: %v", hasUsage) - return hasUsage -} diff --git a/pkg/workflow/compiler_orchestrator_engine.go b/pkg/workflow/compiler_orchestrator_engine.go new file mode 100644 index 0000000000..f8d2f14c1e --- /dev/null +++ b/pkg/workflow/compiler_orchestrator_engine.go @@ -0,0 +1,239 @@ +package workflow + +import ( + "fmt" + "os" + + "github.com/githubnext/gh-aw/pkg/console" + "github.com/githubnext/gh-aw/pkg/logger" + "github.com/githubnext/gh-aw/pkg/parser" +) + +var orchestratorEngineLog = logger.New("workflow:compiler_orchestrator_engine") + +// engineSetupResult holds the results of engine configuration and validation +type engineSetupResult struct { + engineSetting string + engineConfig *EngineConfig + agenticEngine CodingAgentEngine + networkPermissions *NetworkPermissions + sandboxConfig *SandboxConfig + importsResult *parser.ImportsResult +} + +// setupEngineAndImports configures the AI engine, processes imports, and validates network/sandbox settings. +// This function handles: +// - Engine extraction and validation +// - Import processing and merging +// - Network permissions setup +// - Sandbox configuration +// - Strict mode validations +func (c *Compiler) setupEngineAndImports(result *parser.FrontmatterResult, cleanPath string, content []byte, markdownDir string) (*engineSetupResult, error) { + orchestratorEngineLog.Printf("Setting up engine and processing imports") + + // Extract AI engine setting from frontmatter + engineSetting, engineConfig := c.ExtractEngineConfig(result.Frontmatter) + + // Extract network permissions from frontmatter + networkPermissions := c.extractNetworkPermissions(result.Frontmatter) + + // Default to 'defaults' ecosystem if no network permissions specified + if networkPermissions == nil { + networkPermissions = &NetworkPermissions{ + Allowed: []string{"defaults"}, + } + } + + // Extract sandbox configuration from frontmatter + sandboxConfig := c.extractSandboxConfig(result.Frontmatter) + + // Save the initial strict mode state to restore it after this workflow is processed + // This ensures that strict mode from one workflow doesn't affect other workflows + initialStrictMode := c.strictMode + + // Check strict mode in frontmatter + // Priority: CLI flag > frontmatter > schema default (true) + if !c.strictMode { + // CLI flag not set, check frontmatter + if strictValue, exists := result.Frontmatter["strict"]; exists { + // Frontmatter explicitly sets strict mode + if strictBool, ok := strictValue.(bool); ok { + c.strictMode = strictBool + } + } else { + // Neither CLI nor frontmatter set - use schema default (true) + c.strictMode = true + } + } + + // Perform strict mode validations + orchestratorEngineLog.Printf("Performing strict mode validation (strict=%v)", c.strictMode) + if err := c.validateStrictMode(result.Frontmatter, networkPermissions); err != nil { + orchestratorEngineLog.Printf("Strict mode validation failed: %v", err) + // Restore strict mode before returning error + c.strictMode = initialStrictMode + return nil, err + } + + // Restore the initial strict mode state after validation + // This ensures strict mode doesn't leak to other workflows being compiled + c.strictMode = initialStrictMode + + // Override with command line AI engine setting if provided + if c.engineOverride != "" { + originalEngineSetting := engineSetting + if originalEngineSetting != "" && originalEngineSetting != c.engineOverride { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Command line --engine %s overrides markdown file engine: %s", c.engineOverride, originalEngineSetting))) + c.IncrementWarningCount() + } + engineSetting = c.engineOverride + } + + // Process imports from frontmatter first (before @include directives) + orchestratorEngineLog.Printf("Processing imports from frontmatter") + importCache := c.getSharedImportCache() + // Pass the full file content for accurate line/column error reporting + importsResult, err := parser.ProcessImportsFromFrontmatterWithSource(result.Frontmatter, markdownDir, importCache, cleanPath, string(content)) + if err != nil { + orchestratorEngineLog.Printf("Import processing failed: %v", err) + return nil, err // Error is already formatted with source location + } + + // Merge network permissions from imports with top-level network permissions + if importsResult.MergedNetwork != "" { + orchestratorEngineLog.Printf("Merging network permissions from imports") + networkPermissions, err = c.MergeNetworkPermissions(networkPermissions, importsResult.MergedNetwork) + if err != nil { + orchestratorEngineLog.Printf("Network permissions merge failed: %v", err) + return nil, fmt.Errorf("failed to merge network permissions: %w", err) + } + } + + // Validate permissions from imports against top-level permissions + // Extract top-level permissions first + topLevelPermissions := c.extractPermissions(result.Frontmatter) + if importsResult.MergedPermissions != "" { + orchestratorEngineLog.Printf("Validating included permissions") + if err := c.ValidateIncludedPermissions(topLevelPermissions, importsResult.MergedPermissions); err != nil { + orchestratorEngineLog.Printf("Included permissions validation failed: %v", err) + return nil, fmt.Errorf("permission validation failed: %w", err) + } + } + + // Process @include directives to extract engine configurations and check for conflicts + orchestratorEngineLog.Printf("Expanding includes for engine configurations") + includedEngines, err := parser.ExpandIncludesForEngines(result.Markdown, markdownDir) + if err != nil { + orchestratorEngineLog.Printf("Failed to expand includes for engines: %v", err) + return nil, fmt.Errorf("failed to expand includes for engines: %w", err) + } + + // Combine imported engines with included engines + allEngines := append(importsResult.MergedEngines, includedEngines...) + + // Validate that only one engine field exists across all files + orchestratorEngineLog.Printf("Validating single engine specification") + finalEngineSetting, err := c.validateSingleEngineSpecification(engineSetting, allEngines) + if err != nil { + orchestratorEngineLog.Printf("Engine specification validation failed: %v", err) + return nil, err + } + if finalEngineSetting != "" { + engineSetting = finalEngineSetting + } + + // If engineConfig is nil (engine was in an included file), extract it from the included engine JSON + if engineConfig == nil && len(allEngines) > 0 { + orchestratorEngineLog.Printf("Extracting engine config from included file") + extractedConfig, err := c.extractEngineConfigFromJSON(allEngines[0]) + if err != nil { + orchestratorEngineLog.Printf("Failed to extract engine config: %v", err) + return nil, fmt.Errorf("failed to extract engine config from included file: %w", err) + } + engineConfig = extractedConfig + } + + // Apply the default AI engine setting if not specified + if engineSetting == "" { + defaultEngine := c.engineRegistry.GetDefaultEngine() + engineSetting = defaultEngine.GetID() + log.Printf("No 'engine:' setting found, defaulting to: %s", engineSetting) + // Create a default EngineConfig with the default engine ID if not already set + if engineConfig == nil { + engineConfig = &EngineConfig{ID: engineSetting} + } else if engineConfig.ID == "" { + engineConfig.ID = engineSetting + } + } + + // Validate the engine setting + orchestratorEngineLog.Printf("Validating engine setting: %s", engineSetting) + if err := c.validateEngine(engineSetting); err != nil { + orchestratorEngineLog.Printf("Engine validation failed: %v", err) + return nil, err + } + + // Get the agentic engine instance + agenticEngine, err := c.getAgenticEngine(engineSetting) + if err != nil { + orchestratorEngineLog.Printf("Failed to get agentic engine: %v", err) + return nil, err + } + + log.Printf("AI engine: %s (%s)", agenticEngine.GetDisplayName(), engineSetting) + if agenticEngine.IsExperimental() && c.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Using experimental engine: %s", agenticEngine.GetDisplayName()))) + c.IncrementWarningCount() + } + + // Enable firewall by default for copilot engine when network restrictions are present + // (unless SRT sandbox is configured, since AWF and SRT are mutually exclusive) + enableFirewallByDefaultForCopilot(engineSetting, networkPermissions, sandboxConfig) + + // Enable firewall by default for claude engine when network restrictions are present + enableFirewallByDefaultForClaude(engineSetting, networkPermissions, sandboxConfig) + + // Re-evaluate strict mode for firewall and network validation + // (it was restored after validateStrictMode but we need it again) + initialStrictModeForFirewall := c.strictMode + if !c.strictMode { + // CLI flag not set, check frontmatter + if strictValue, exists := result.Frontmatter["strict"]; exists { + // Frontmatter explicitly sets strict mode + if strictBool, ok := strictValue.(bool); ok { + c.strictMode = strictBool + } + } else { + // Neither CLI nor frontmatter set - use schema default (true) + c.strictMode = true + } + } + + // Validate firewall is enabled in strict mode for copilot with network restrictions + orchestratorEngineLog.Printf("Validating strict firewall (strict=%v)", c.strictMode) + if err := c.validateStrictFirewall(engineSetting, networkPermissions, sandboxConfig); err != nil { + orchestratorEngineLog.Printf("Strict firewall validation failed: %v", err) + c.strictMode = initialStrictModeForFirewall + return nil, err + } + + // Check if the engine supports network restrictions when they are defined + if err := c.checkNetworkSupport(agenticEngine, networkPermissions); err != nil { + orchestratorEngineLog.Printf("Network support check failed: %v", err) + // Restore strict mode before returning error + c.strictMode = initialStrictModeForFirewall + return nil, err + } + + // Restore the strict mode state after network check + c.strictMode = initialStrictModeForFirewall + + return &engineSetupResult{ + engineSetting: engineSetting, + engineConfig: engineConfig, + agenticEngine: agenticEngine, + networkPermissions: networkPermissions, + sandboxConfig: sandboxConfig, + importsResult: importsResult, + }, nil +} diff --git a/pkg/workflow/compiler_orchestrator_frontmatter.go b/pkg/workflow/compiler_orchestrator_frontmatter.go new file mode 100644 index 0000000000..f0fb593b18 --- /dev/null +++ b/pkg/workflow/compiler_orchestrator_frontmatter.go @@ -0,0 +1,167 @@ +package workflow + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/githubnext/gh-aw/pkg/logger" + "github.com/githubnext/gh-aw/pkg/parser" +) + +var orchestratorFrontmatterLog = logger.New("workflow:compiler_orchestrator_frontmatter") + +// frontmatterParseResult holds the results of parsing and validating frontmatter +type frontmatterParseResult struct { + cleanPath string + content []byte + frontmatterResult *parser.FrontmatterResult + frontmatterForValidation map[string]any + markdownDir string + isSharedWorkflow bool +} + +// parseFrontmatterSection reads the workflow file and parses its frontmatter. +// It returns a frontmatterParseResult containing the parsed data and validation information. +// If the workflow is detected as a shared workflow (no 'on' field), isSharedWorkflow is set to true. +func (c *Compiler) parseFrontmatterSection(markdownPath string) (*frontmatterParseResult, error) { + orchestratorFrontmatterLog.Printf("Starting frontmatter parsing: %s", markdownPath) + log.Printf("Reading file: %s", markdownPath) + + // Clean the path to prevent path traversal issues (gosec G304) + // filepath.Clean removes ".." and other problematic path elements + cleanPath := filepath.Clean(markdownPath) + + // Read the file + content, err := os.ReadFile(cleanPath) + if err != nil { + orchestratorFrontmatterLog.Printf("Failed to read file: %s, error: %v", cleanPath, err) + return nil, fmt.Errorf("failed to read file: %w", err) + } + + log.Printf("File size: %d bytes", len(content)) + + // Parse frontmatter and markdown + orchestratorFrontmatterLog.Printf("Parsing frontmatter from file: %s", cleanPath) + result, err := parser.ExtractFrontmatterFromContent(string(content)) + if err != nil { + orchestratorFrontmatterLog.Printf("Frontmatter extraction failed: %v", err) + // Use FrontmatterStart from result if available, otherwise default to line 2 (after opening ---) + frontmatterStart := 2 + if result != nil && result.FrontmatterStart > 0 { + frontmatterStart = result.FrontmatterStart + } + return nil, c.createFrontmatterError(cleanPath, string(content), err, frontmatterStart) + } + + if len(result.Frontmatter) == 0 { + orchestratorFrontmatterLog.Print("No frontmatter found in file") + return nil, fmt.Errorf("no frontmatter found") + } + + // Preprocess schedule fields to convert human-friendly format to cron expressions + if err := c.preprocessScheduleFields(result.Frontmatter, cleanPath, string(content)); err != nil { + orchestratorFrontmatterLog.Printf("Schedule preprocessing failed: %v", err) + return nil, err + } + + // Create a copy of frontmatter without internal markers for schema validation + // Keep the original frontmatter with markers for YAML generation + frontmatterForValidation := c.copyFrontmatterWithoutInternalMarkers(result.Frontmatter) + + // Check if "on" field is missing - if so, treat as a shared/imported workflow + _, hasOnField := frontmatterForValidation["on"] + if !hasOnField { + detectionLog.Printf("No 'on' field detected - treating as shared agentic workflow") + + // Validate as an included/shared workflow (uses main_workflow_schema with forbidden field checks) + if err := parser.ValidateIncludedFileFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { + orchestratorFrontmatterLog.Printf("Shared workflow validation failed: %v", err) + return nil, err + } + + return &frontmatterParseResult{ + cleanPath: cleanPath, + content: content, + frontmatterResult: result, + frontmatterForValidation: frontmatterForValidation, + markdownDir: filepath.Dir(cleanPath), + isSharedWorkflow: true, + }, nil + } + + // For main workflows (with 'on' field), markdown content is required + if result.Markdown == "" { + orchestratorFrontmatterLog.Print("No markdown content found for main workflow") + return nil, fmt.Errorf("no markdown content found") + } + + // Validate main workflow frontmatter contains only expected entries + orchestratorFrontmatterLog.Printf("Validating main workflow frontmatter schema") + if err := parser.ValidateMainWorkflowFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { + orchestratorFrontmatterLog.Printf("Main workflow frontmatter validation failed: %v", err) + return nil, err + } + + // Validate event filter mutual exclusivity (branches/branches-ignore, paths/paths-ignore) + if err := ValidateEventFilters(frontmatterForValidation); err != nil { + orchestratorFrontmatterLog.Printf("Event filter validation failed: %v", err) + return nil, err + } + + // Validate that @include/@import directives are not used inside template regions + if err := validateNoIncludesInTemplateRegions(result.Markdown); err != nil { + orchestratorFrontmatterLog.Printf("Template region validation failed: %v", err) + return nil, fmt.Errorf("template region validation failed: %w", err) + } + + log.Printf("Frontmatter: %d chars, Markdown: %d chars", len(result.Frontmatter), len(result.Markdown)) + + return &frontmatterParseResult{ + cleanPath: cleanPath, + content: content, + frontmatterResult: result, + frontmatterForValidation: frontmatterForValidation, + markdownDir: filepath.Dir(cleanPath), + isSharedWorkflow: false, + }, nil +} + +// copyFrontmatterWithoutInternalMarkers creates a deep copy of frontmatter without internal marker fields +// This is used for schema validation while preserving markers in the original for YAML generation +func (c *Compiler) copyFrontmatterWithoutInternalMarkers(frontmatter map[string]any) map[string]any { + // Create a shallow copy of the top level + copy := make(map[string]any) + for k, v := range frontmatter { + if k == "on" { + // Special handling for "on" field - need to deep copy and remove markers + if onMap, ok := v.(map[string]any); ok { + onCopy := make(map[string]any) + for onKey, onValue := range onMap { + if onKey == "issues" || onKey == "pull_request" || onKey == "discussion" { + // Deep copy the section and remove marker + if sectionMap, ok := onValue.(map[string]any); ok { + sectionCopy := make(map[string]any) + for sectionKey, sectionValue := range sectionMap { + if sectionKey != "__gh_aw_native_label_filter__" { + sectionCopy[sectionKey] = sectionValue + } + } + onCopy[onKey] = sectionCopy + } else { + onCopy[onKey] = onValue + } + } else { + onCopy[onKey] = onValue + } + } + copy[k] = onCopy + } else { + copy[k] = v + } + } else { + copy[k] = v + } + } + return copy +} diff --git a/pkg/workflow/compiler_orchestrator_tools.go b/pkg/workflow/compiler_orchestrator_tools.go new file mode 100644 index 0000000000..6f480b6ba8 --- /dev/null +++ b/pkg/workflow/compiler_orchestrator_tools.go @@ -0,0 +1,258 @@ +package workflow + +import ( + "fmt" + "os" + "sort" + "strings" + + "github.com/githubnext/gh-aw/pkg/console" + "github.com/githubnext/gh-aw/pkg/logger" + "github.com/githubnext/gh-aw/pkg/parser" +) + +var orchestratorToolsLog = logger.New("workflow:compiler_orchestrator_tools") + +// toolsProcessingResult holds the results of tools and markdown processing +type toolsProcessingResult struct { + tools map[string]any + runtimes map[string]any + toolsTimeout int + toolsStartupTimeout int + markdownContent string + allIncludedFiles []string + workflowName string + frontmatterName string + needsTextOutput bool + trackerID string + safeOutputs *SafeOutputsConfig + secretMasking *SecretMaskingConfig + parsedFrontmatter *FrontmatterConfig +} + +// processToolsAndMarkdown processes tools configuration, runtimes, and markdown content. +// This function handles: +// - Safe outputs and secret masking configuration +// - Tools and MCP servers merging +// - Runtimes merging +// - MCP validations +// - Markdown content expansion +// - Workflow name extraction +func (c *Compiler) processToolsAndMarkdown(result *parser.FrontmatterResult, cleanPath string, markdownDir string, + agenticEngine CodingAgentEngine, engineSetting string, importsResult *parser.ImportsResult) (*toolsProcessingResult, error) { + + orchestratorToolsLog.Printf("Processing tools and markdown") + log.Print("Processing tools and includes...") + + // Extract SafeOutputs configuration early so we can use it when applying default tools + safeOutputs := c.extractSafeOutputsConfig(result.Frontmatter) + + // Extract SecretMasking configuration + secretMasking := c.extractSecretMaskingConfig(result.Frontmatter) + + // Merge secret-masking from imports with top-level secret-masking + if importsResult.MergedSecretMasking != "" { + orchestratorToolsLog.Printf("Merging secret-masking from imports") + var err error + secretMasking, err = c.MergeSecretMasking(secretMasking, importsResult.MergedSecretMasking) + if err != nil { + orchestratorToolsLog.Printf("Secret-masking merge failed: %v", err) + return nil, fmt.Errorf("failed to merge secret-masking: %w", err) + } + } + + var tools map[string]any + + // Extract tools from the main file + topTools := extractToolsFromFrontmatter(result.Frontmatter) + + // Extract mcp-servers from the main file and merge them into tools + mcpServers := extractMCPServersFromFrontmatter(result.Frontmatter) + + // Process @include directives to extract additional tools + orchestratorToolsLog.Printf("Expanding includes for tools") + includedTools, includedToolFiles, err := parser.ExpandIncludesWithManifest(result.Markdown, markdownDir, true) + if err != nil { + orchestratorToolsLog.Printf("Failed to expand includes for tools: %v", err) + return nil, fmt.Errorf("failed to expand includes for tools: %w", err) + } + + // Combine imported tools with included tools + var allIncludedTools string + if importsResult.MergedTools != "" && includedTools != "" { + allIncludedTools = importsResult.MergedTools + "\n" + includedTools + } else if importsResult.MergedTools != "" { + allIncludedTools = importsResult.MergedTools + } else { + allIncludedTools = includedTools + } + + // Combine imported mcp-servers with top-level mcp-servers + // Imported mcp-servers are in JSON format (newline-separated), need to merge them + allMCPServers := mcpServers + if importsResult.MergedMCPServers != "" { + orchestratorToolsLog.Printf("Merging imported mcp-servers") + // Parse and merge imported MCP servers + mergedMCPServers, err := c.MergeMCPServers(mcpServers, importsResult.MergedMCPServers) + if err != nil { + orchestratorToolsLog.Printf("MCP servers merge failed: %v", err) + return nil, fmt.Errorf("failed to merge imported mcp-servers: %w", err) + } + allMCPServers = mergedMCPServers + } + + // Merge tools including mcp-servers + orchestratorToolsLog.Printf("Merging tools and MCP servers") + tools, err = c.mergeToolsAndMCPServers(topTools, allMCPServers, allIncludedTools) + if err != nil { + orchestratorToolsLog.Printf("Tools merge failed: %v", err) + return nil, fmt.Errorf("failed to merge tools: %w", err) + } + + // Extract and validate tools timeout settings + toolsTimeout, err := c.extractToolsTimeout(tools) + if err != nil { + return nil, fmt.Errorf("invalid tools timeout configuration: %w", err) + } + + toolsStartupTimeout, err := c.extractToolsStartupTimeout(tools) + if err != nil { + return nil, fmt.Errorf("invalid tools startup timeout configuration: %w", err) + } + + // Remove meta fields (timeout, startup-timeout) from merged tools map + // These are configuration fields, not actual tools + delete(tools, "timeout") + delete(tools, "startup-timeout") + + // Extract and merge runtimes from frontmatter and imports + topRuntimes := extractRuntimesFromFrontmatter(result.Frontmatter) + orchestratorToolsLog.Printf("Merging runtimes") + runtimes, err := mergeRuntimes(topRuntimes, importsResult.MergedRuntimes) + if err != nil { + orchestratorToolsLog.Printf("Runtimes merge failed: %v", err) + return nil, fmt.Errorf("failed to merge runtimes: %w", err) + } + + // Add MCP fetch server if needed (when web-fetch is requested but engine doesn't support it) + tools, _ = AddMCPFetchServerIfNeeded(tools, agenticEngine) + + // Validate MCP configurations + orchestratorToolsLog.Printf("Validating MCP configurations") + if err := ValidateMCPConfigs(tools); err != nil { + orchestratorToolsLog.Printf("MCP configuration validation failed: %v", err) + return nil, err + } + + // Validate HTTP transport support for the current engine + if err := c.validateHTTPTransportSupport(tools, agenticEngine); err != nil { + orchestratorToolsLog.Printf("HTTP transport validation failed: %v", err) + return nil, err + } + + if !agenticEngine.SupportsToolsAllowlist() { + // For engines that don't support tool allowlists (like codex), ignore tools section and provide warnings + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Using experimental %s support (engine: %s)", agenticEngine.GetDisplayName(), engineSetting))) + c.IncrementWarningCount() + if _, hasTools := result.Frontmatter["tools"]; hasTools { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("'tools' section ignored when using engine: %s (%s doesn't support MCP tool allow-listing)", engineSetting, agenticEngine.GetDisplayName()))) + c.IncrementWarningCount() + } + tools = map[string]any{} + // For now, we'll add a basic github tool (always uses docker MCP) + githubConfig := map[string]any{} + tools["github"] = githubConfig + } + + // Validate max-turns support for the current engine + if err := c.validateMaxTurnsSupport(result.Frontmatter, agenticEngine); err != nil { + return nil, err + } + + // Validate web-search support for the current engine (warning only) + c.validateWebSearchSupport(tools, agenticEngine) + + // Process @include directives in markdown content + markdownContent, includedMarkdownFiles, err := parser.ExpandIncludesWithManifest(result.Markdown, markdownDir, false) + if err != nil { + return nil, fmt.Errorf("failed to expand includes in markdown: %w", err) + } + + // Prepend imported markdown from frontmatter imports field + if importsResult.MergedMarkdown != "" { + markdownContent = importsResult.MergedMarkdown + markdownContent + } + + log.Print("Expanded includes in markdown content") + + // Combine all included files (from tools and markdown) + // Use a map to deduplicate files + allIncludedFilesMap := make(map[string]bool) + for _, file := range includedToolFiles { + allIncludedFilesMap[file] = true + } + for _, file := range includedMarkdownFiles { + allIncludedFilesMap[file] = true + } + var allIncludedFiles []string + for file := range allIncludedFilesMap { + allIncludedFiles = append(allIncludedFiles, file) + } + // Sort files alphabetically to ensure consistent ordering in lock files + sort.Strings(allIncludedFiles) + + // Extract workflow name + workflowName, err := parser.ExtractWorkflowNameFromMarkdown(cleanPath) + if err != nil { + return nil, fmt.Errorf("failed to extract workflow name: %w", err) + } + + // Check if frontmatter specifies a custom name and use it instead + frontmatterName := extractStringFromMap(result.Frontmatter, "name", nil) + if frontmatterName != "" { + workflowName = frontmatterName + } + + log.Printf("Extracted workflow name: '%s'", workflowName) + + // Check if the markdown content uses the text output + needsTextOutput := c.detectTextOutputUsage(markdownContent) + + // Extract and validate tracker-id + trackerID, err := c.extractTrackerID(result.Frontmatter) + if err != nil { + return nil, err + } + + // Parse frontmatter config once for performance optimization + parsedFrontmatter, err := ParseFrontmatterConfig(result.Frontmatter) + if err != nil { + orchestratorToolsLog.Printf("Failed to parse frontmatter config: %v", err) + // Non-fatal error - continue with nil ParsedFrontmatter + parsedFrontmatter = nil + } + + return &toolsProcessingResult{ + tools: tools, + runtimes: runtimes, + toolsTimeout: toolsTimeout, + toolsStartupTimeout: toolsStartupTimeout, + markdownContent: markdownContent, + allIncludedFiles: allIncludedFiles, + workflowName: workflowName, + frontmatterName: frontmatterName, + needsTextOutput: needsTextOutput, + trackerID: trackerID, + safeOutputs: safeOutputs, + secretMasking: secretMasking, + parsedFrontmatter: parsedFrontmatter, + }, nil +} + +// detectTextOutputUsage checks if the markdown content uses ${{ needs.activation.outputs.text }} +func (c *Compiler) detectTextOutputUsage(markdownContent string) bool { + // Check for the specific GitHub Actions expression + hasUsage := strings.Contains(markdownContent, "${{ needs.activation.outputs.text }}") + detectionLog.Printf("Detected usage of activation.outputs.text: %v", hasUsage) + return hasUsage +} diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go new file mode 100644 index 0000000000..26b17088c9 --- /dev/null +++ b/pkg/workflow/compiler_orchestrator_workflow.go @@ -0,0 +1,450 @@ +package workflow + +import ( + "fmt" + "os" + "strings" + + "github.com/githubnext/gh-aw/pkg/console" + "github.com/githubnext/gh-aw/pkg/logger" + "github.com/githubnext/gh-aw/pkg/parser" + "github.com/goccy/go-yaml" +) + +var orchestratorWorkflowLog = logger.New("workflow:compiler_orchestrator_workflow") + +// ParseWorkflowFile parses a workflow markdown file and returns a WorkflowData structure. +// This is the main orchestration function that coordinates all compilation phases. +func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) { + orchestratorWorkflowLog.Printf("Starting workflow file parsing: %s", markdownPath) + + // Parse frontmatter section + parseResult, err := c.parseFrontmatterSection(markdownPath) + if err != nil { + return nil, err + } + + // Handle shared workflows + if parseResult.isSharedWorkflow { + return nil, &SharedWorkflowError{Path: parseResult.cleanPath} + } + + // Unpack parse result for convenience + cleanPath := parseResult.cleanPath + content := parseResult.content + result := parseResult.frontmatterResult + markdownDir := parseResult.markdownDir + + // Setup engine and process imports + engineSetup, err := c.setupEngineAndImports(result, cleanPath, content, markdownDir) + if err != nil { + return nil, err + } + + // Process tools and markdown + toolsResult, err := c.processToolsAndMarkdown(result, cleanPath, markdownDir, engineSetup.agenticEngine, engineSetup.engineSetting, engineSetup.importsResult) + if err != nil { + return nil, err + } + + // Build initial workflow data structure + workflowData := c.buildInitialWorkflowData(result, toolsResult, engineSetup, engineSetup.importsResult) + + // Use shared action cache and resolver from the compiler + actionCache, actionResolver := c.getSharedActionResolver() + workflowData.ActionCache = actionCache + workflowData.ActionResolver = actionResolver + workflowData.ActionPinWarnings = c.actionPinWarnings + + // Extract YAML configuration sections from frontmatter + c.extractYAMLSections(result.Frontmatter, workflowData) + + // Process and merge custom steps with imported steps + c.processAndMergeSteps(result.Frontmatter, workflowData, engineSetup.importsResult) + + // Process and merge post-steps + c.processAndMergePostSteps(result.Frontmatter, workflowData) + + // Process and merge services + c.processAndMergeServices(result.Frontmatter, workflowData, engineSetup.importsResult) + + // Extract additional configurations (cache, safe-inputs, safe-outputs, etc.) + if err := c.extractAdditionalConfigurations( + result.Frontmatter, + toolsResult.tools, + markdownDir, + workflowData, + engineSetup.importsResult, + result.Markdown, + toolsResult.safeOutputs, + ); err != nil { + return nil, err + } + + // Process on section configuration and apply filters + if err := c.processOnSectionAndFilters(result.Frontmatter, workflowData, cleanPath); err != nil { + return nil, err + } + + orchestratorWorkflowLog.Printf("Workflow file parsing completed successfully: %s", markdownPath) + return workflowData, nil +} + +// buildInitialWorkflowData creates the initial WorkflowData struct with basic fields populated +func (c *Compiler) buildInitialWorkflowData( + result *parser.FrontmatterResult, + toolsResult *toolsProcessingResult, + engineSetup *engineSetupResult, + importsResult *parser.ImportsResult, +) *WorkflowData { + orchestratorWorkflowLog.Print("Building initial workflow data") + + return &WorkflowData{ + Name: toolsResult.workflowName, + FrontmatterName: toolsResult.frontmatterName, + FrontmatterYAML: strings.Join(result.FrontmatterLines, "\n"), + Description: c.extractDescription(result.Frontmatter), + Source: c.extractSource(result.Frontmatter), + TrackerID: toolsResult.trackerID, + ImportedFiles: importsResult.ImportedFiles, + IncludedFiles: toolsResult.allIncludedFiles, + ImportInputs: importsResult.ImportInputs, + Tools: toolsResult.tools, + ParsedTools: NewTools(toolsResult.tools), + Runtimes: toolsResult.runtimes, + MarkdownContent: toolsResult.markdownContent, + AI: engineSetup.engineSetting, + EngineConfig: engineSetup.engineConfig, + AgentFile: importsResult.AgentFile, + NetworkPermissions: engineSetup.networkPermissions, + SandboxConfig: applySandboxDefaults(engineSetup.sandboxConfig, engineSetup.engineConfig), + NeedsTextOutput: toolsResult.needsTextOutput, + ToolsTimeout: toolsResult.toolsTimeout, + ToolsStartupTimeout: toolsResult.toolsStartupTimeout, + TrialMode: c.trialMode, + TrialLogicalRepo: c.trialLogicalRepoSlug, + GitHubToken: extractStringFromMap(result.Frontmatter, "github-token", nil), + StrictMode: c.strictMode, + SecretMasking: toolsResult.secretMasking, + ParsedFrontmatter: toolsResult.parsedFrontmatter, + } +} + +// extractYAMLSections extracts YAML configuration sections from frontmatter +func (c *Compiler) extractYAMLSections(frontmatter map[string]any, workflowData *WorkflowData) { + orchestratorWorkflowLog.Print("Extracting YAML sections from frontmatter") + + workflowData.On = c.extractTopLevelYAMLSection(frontmatter, "on") + workflowData.Permissions = c.extractPermissions(frontmatter) + workflowData.Network = c.extractTopLevelYAMLSection(frontmatter, "network") + workflowData.Concurrency = c.extractTopLevelYAMLSection(frontmatter, "concurrency") + workflowData.RunName = c.extractTopLevelYAMLSection(frontmatter, "run-name") + workflowData.Env = c.extractTopLevelYAMLSection(frontmatter, "env") + workflowData.Features = c.extractFeatures(frontmatter) + workflowData.If = c.extractIfCondition(frontmatter) + + // Prefer timeout-minutes (new) over timeout_minutes (deprecated) + workflowData.TimeoutMinutes = c.extractTopLevelYAMLSection(frontmatter, "timeout-minutes") + if workflowData.TimeoutMinutes == "" { + workflowData.TimeoutMinutes = c.extractTopLevelYAMLSection(frontmatter, "timeout_minutes") + if workflowData.TimeoutMinutes != "" { + // Emit deprecation warning + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Field 'timeout_minutes' is deprecated. Please use 'timeout-minutes' instead to follow GitHub Actions naming convention.")) + } + } + + workflowData.RunsOn = c.extractTopLevelYAMLSection(frontmatter, "runs-on") + workflowData.Environment = c.extractTopLevelYAMLSection(frontmatter, "environment") + workflowData.Container = c.extractTopLevelYAMLSection(frontmatter, "container") + workflowData.Cache = c.extractTopLevelYAMLSection(frontmatter, "cache") +} + +// processAndMergeSteps handles the merging of imported steps with main workflow steps +func (c *Compiler) processAndMergeSteps(frontmatter map[string]any, workflowData *WorkflowData, importsResult *parser.ImportsResult) { + orchestratorWorkflowLog.Print("Processing and merging custom steps") + + workflowData.CustomSteps = c.extractTopLevelYAMLSection(frontmatter, "steps") + + // Merge imported steps if any + if importsResult.MergedSteps != "" { + // Parse imported steps from YAML array + var importedSteps []any + if err := yaml.Unmarshal([]byte(importsResult.MergedSteps), &importedSteps); err == nil { + // Apply action pinning to imported steps + importedSteps = ApplyActionPinsToSteps(importedSteps, workflowData) + + // If there are main workflow steps, parse and merge them + if workflowData.CustomSteps != "" { + // Parse main workflow steps (format: "steps:\n - ...") + var mainStepsWrapper map[string]any + if err := yaml.Unmarshal([]byte(workflowData.CustomSteps), &mainStepsWrapper); err == nil { + if mainStepsVal, hasSteps := mainStepsWrapper["steps"]; hasSteps { + if mainSteps, ok := mainStepsVal.([]any); ok { + // Apply action pinning to main steps + mainSteps = ApplyActionPinsToSteps(mainSteps, workflowData) + + // Prepend imported steps to main steps + allSteps := append(importedSteps, mainSteps...) + // Convert back to YAML with "steps:" wrapper + stepsWrapper := map[string]any{"steps": allSteps} + stepsYAML, err := yaml.Marshal(stepsWrapper) + if err == nil { + // Remove quotes from uses values with version comments + workflowData.CustomSteps = unquoteUsesWithComments(string(stepsYAML)) + } + } + } + } + } else { + // Only imported steps exist, wrap in "steps:" format + stepsWrapper := map[string]any{"steps": importedSteps} + stepsYAML, err := yaml.Marshal(stepsWrapper) + if err == nil { + // Remove quotes from uses values with version comments + workflowData.CustomSteps = unquoteUsesWithComments(string(stepsYAML)) + } + } + } + } else if workflowData.CustomSteps != "" { + // No imported steps, but there are main steps - still apply pinning + var mainStepsWrapper map[string]any + if err := yaml.Unmarshal([]byte(workflowData.CustomSteps), &mainStepsWrapper); err == nil { + if mainStepsVal, hasSteps := mainStepsWrapper["steps"]; hasSteps { + if mainSteps, ok := mainStepsVal.([]any); ok { + // Apply action pinning to main steps + mainSteps = ApplyActionPinsToSteps(mainSteps, workflowData) + + // Convert back to YAML with "steps:" wrapper + stepsWrapper := map[string]any{"steps": mainSteps} + stepsYAML, err := yaml.Marshal(stepsWrapper) + if err == nil { + // Remove quotes from uses values with version comments + workflowData.CustomSteps = unquoteUsesWithComments(string(stepsYAML)) + } + } + } + } + } +} + +// processAndMergePostSteps handles the processing of post-steps with action pinning +func (c *Compiler) processAndMergePostSteps(frontmatter map[string]any, workflowData *WorkflowData) { + orchestratorWorkflowLog.Print("Processing post-steps") + + workflowData.PostSteps = c.extractTopLevelYAMLSection(frontmatter, "post-steps") + + // Apply action pinning to post-steps if any + if workflowData.PostSteps != "" { + var postStepsWrapper map[string]any + if err := yaml.Unmarshal([]byte(workflowData.PostSteps), &postStepsWrapper); err == nil { + if postStepsVal, hasPostSteps := postStepsWrapper["post-steps"]; hasPostSteps { + if postSteps, ok := postStepsVal.([]any); ok { + // Apply action pinning to post steps + postSteps = ApplyActionPinsToSteps(postSteps, workflowData) + + // Convert back to YAML with "post-steps:" wrapper + stepsWrapper := map[string]any{"post-steps": postSteps} + stepsYAML, err := yaml.Marshal(stepsWrapper) + if err == nil { + // Remove quotes from uses values with version comments + workflowData.PostSteps = unquoteUsesWithComments(string(stepsYAML)) + } + } + } + } + } +} + +// processAndMergeServices handles the merging of imported services with main workflow services +func (c *Compiler) processAndMergeServices(frontmatter map[string]any, workflowData *WorkflowData, importsResult *parser.ImportsResult) { + orchestratorWorkflowLog.Print("Processing and merging services") + + workflowData.Services = c.extractTopLevelYAMLSection(frontmatter, "services") + + // Merge imported services if any + if importsResult.MergedServices != "" { + // Parse imported services from YAML + var importedServices map[string]any + if err := yaml.Unmarshal([]byte(importsResult.MergedServices), &importedServices); err == nil { + // If there are main workflow services, parse and merge them + if workflowData.Services != "" { + // Parse main workflow services + var mainServicesWrapper map[string]any + if err := yaml.Unmarshal([]byte(workflowData.Services), &mainServicesWrapper); err == nil { + if mainServices, ok := mainServicesWrapper["services"].(map[string]any); ok { + // Merge: main workflow services take precedence over imported + for key, value := range importedServices { + if _, exists := mainServices[key]; !exists { + mainServices[key] = value + } + } + // Convert back to YAML with "services:" wrapper + servicesWrapper := map[string]any{"services": mainServices} + servicesYAML, err := yaml.Marshal(servicesWrapper) + if err == nil { + workflowData.Services = string(servicesYAML) + } + } + } + } else { + // Only imported services exist, wrap in "services:" format + servicesWrapper := map[string]any{"services": importedServices} + servicesYAML, err := yaml.Marshal(servicesWrapper) + if err == nil { + workflowData.Services = string(servicesYAML) + } + } + } + } +} + +// extractAdditionalConfigurations extracts cache-memory, repo-memory, safe-inputs, and safe-outputs configurations +func (c *Compiler) extractAdditionalConfigurations( + frontmatter map[string]any, + tools map[string]any, + markdownDir string, + workflowData *WorkflowData, + importsResult *parser.ImportsResult, + markdown string, + safeOutputs *SafeOutputsConfig, +) error { + orchestratorWorkflowLog.Print("Extracting additional configurations") + + // Extract cache-memory config and check for errors + cacheMemoryConfig, err := c.extractCacheMemoryConfigFromMap(tools) + if err != nil { + return err + } + workflowData.CacheMemoryConfig = cacheMemoryConfig + + // Extract repo-memory config and check for errors + toolsConfig, err := ParseToolsConfig(tools) + if err != nil { + return err + } + repoMemoryConfig, err := c.extractRepoMemoryConfig(toolsConfig) + if err != nil { + return err + } + workflowData.RepoMemoryConfig = repoMemoryConfig + + // Extract and process safe-inputs and safe-outputs + workflowData.Command, workflowData.CommandEvents = c.extractCommandConfig(frontmatter) + workflowData.Jobs = c.extractJobsFromFrontmatter(frontmatter) + workflowData.Roles = c.extractRoles(frontmatter) + workflowData.Bots = c.extractBots(frontmatter) + + // Use the already extracted output configuration + workflowData.SafeOutputs = safeOutputs + + // Extract safe-inputs configuration + workflowData.SafeInputs = c.extractSafeInputsConfig(frontmatter) + + // Merge safe-inputs from imports + if len(importsResult.MergedSafeInputs) > 0 { + workflowData.SafeInputs = c.mergeSafeInputs(workflowData.SafeInputs, importsResult.MergedSafeInputs) + } + + // Extract safe-jobs from safe-outputs.jobs location + topSafeJobs := extractSafeJobsFromFrontmatter(frontmatter) + + // Process @include directives to extract additional safe-outputs configurations + includedSafeOutputsConfigs, err := parser.ExpandIncludesForSafeOutputs(markdown, markdownDir) + if err != nil { + return fmt.Errorf("failed to expand includes for safe-outputs: %w", err) + } + + // Combine imported safe-outputs with included safe-outputs + var allSafeOutputsConfigs []string + if len(importsResult.MergedSafeOutputs) > 0 { + allSafeOutputsConfigs = append(allSafeOutputsConfigs, importsResult.MergedSafeOutputs...) + } + if len(includedSafeOutputsConfigs) > 0 { + allSafeOutputsConfigs = append(allSafeOutputsConfigs, includedSafeOutputsConfigs...) + } + + // Merge safe-jobs from all safe-outputs configurations (imported and included) + includedSafeJobs, err := c.mergeSafeJobsFromIncludedConfigs(topSafeJobs, allSafeOutputsConfigs) + if err != nil { + return fmt.Errorf("failed to merge safe-jobs from includes: %w", err) + } + + // Merge app configuration from included safe-outputs configurations + includedApp, err := c.mergeAppFromIncludedConfigs(workflowData.SafeOutputs, allSafeOutputsConfigs) + if err != nil { + return fmt.Errorf("failed to merge app from includes: %w", err) + } + + // Ensure SafeOutputs exists and populate the Jobs field with merged jobs + if workflowData.SafeOutputs == nil && len(includedSafeJobs) > 0 { + workflowData.SafeOutputs = &SafeOutputsConfig{} + } + // Always use the merged includedSafeJobs as it contains both main and imported jobs + if workflowData.SafeOutputs != nil && len(includedSafeJobs) > 0 { + workflowData.SafeOutputs.Jobs = includedSafeJobs + } + + // Populate the App field if it's not set in the top-level workflow but is in an included config + if workflowData.SafeOutputs != nil && workflowData.SafeOutputs.App == nil && includedApp != nil { + workflowData.SafeOutputs.App = includedApp + } + + // Merge safe-outputs types from imports + mergedSafeOutputs, err := c.MergeSafeOutputs(workflowData.SafeOutputs, allSafeOutputsConfigs) + if err != nil { + return fmt.Errorf("failed to merge safe-outputs from imports: %w", err) + } + workflowData.SafeOutputs = mergedSafeOutputs + + return nil +} + +// processOnSectionAndFilters processes the on section configuration and applies various filters +func (c *Compiler) processOnSectionAndFilters( + frontmatter map[string]any, + workflowData *WorkflowData, + cleanPath string, +) error { + orchestratorWorkflowLog.Print("Processing on section and filters") + + // Process stop-after configuration from the on: section + if err := c.processStopAfterConfiguration(frontmatter, workflowData, cleanPath); err != nil { + return err + } + + // Process skip-if-match configuration from the on: section + if err := c.processSkipIfMatchConfiguration(frontmatter, workflowData); err != nil { + return err + } + + // Process skip-if-no-match configuration from the on: section + if err := c.processSkipIfNoMatchConfiguration(frontmatter, workflowData); err != nil { + return err + } + + // Process manual-approval configuration from the on: section + if err := c.processManualApprovalConfiguration(frontmatter, workflowData); err != nil { + return err + } + + // Parse the "on" section for command triggers, reactions, and other events + if err := c.parseOnSection(frontmatter, workflowData, cleanPath); err != nil { + return err + } + + // Apply defaults + if err := c.applyDefaults(workflowData, cleanPath); err != nil { + return err + } + + // Apply pull request draft filter if specified + c.applyPullRequestDraftFilter(workflowData, frontmatter) + + // Apply pull request fork filter if specified + c.applyPullRequestForkFilter(workflowData, frontmatter) + + // Apply label filter if specified + c.applyLabelFilter(workflowData, frontmatter) + + return nil +} From 97b70b88da369be1056003e405fab1eb9f15ea29 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:14:29 +0000 Subject: [PATCH 3/3] Fix unused logger and verify split works correctly - Removed unused orchestratorLog from compiler_orchestrator.go - Verified all linting passes - Verified all workflows recompile successfully - All orchestrator tests pass Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/compiler_orchestrator.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/workflow/compiler_orchestrator.go b/pkg/workflow/compiler_orchestrator.go index 70232bf106..ea22fe85eb 100644 --- a/pkg/workflow/compiler_orchestrator.go +++ b/pkg/workflow/compiler_orchestrator.go @@ -1,9 +1,8 @@ package workflow import ( -"github.com/githubnext/gh-aw/pkg/logger" + "github.com/githubnext/gh-aw/pkg/logger" ) -// Shared loggers used across compiler orchestrator modules +// Shared logger used across compiler orchestrator modules var detectionLog = logger.New("workflow:detection") -var orchestratorLog = logger.New("workflow:compiler_orchestrator")