From 31185bbbec3dfa17df4d2dba3d8f591fef317b64 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 24 Jan 2026 04:59:03 +0000 Subject: [PATCH 1/4] Initial plan From 0729dd9f80e2936e6d3180eeabef6761baedb719 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 24 Jan 2026 05:10:09 +0000 Subject: [PATCH 2/4] Phase 1: Extract parseFrontmatterSection function - Extract file reading and frontmatter parsing into parseFrontmatterSection() - Reduces ParseWorkflowFile from ~807 to 737 lines (70 line reduction) - Creates frontmatterParseResult struct for intermediate data - All existing tests pass Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/compiler_orchestrator.go | 78 +++++++++++++++++++++------ 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/pkg/workflow/compiler_orchestrator.go b/pkg/workflow/compiler_orchestrator.go index c3db18943d..e84625e743 100644 --- a/pkg/workflow/compiler_orchestrator.go +++ b/pkg/workflow/compiler_orchestrator.go @@ -16,8 +16,21 @@ import ( var detectionLog = logger.New("workflow:detection") var orchestratorLog = logger.New("workflow:compiler_orchestrator") -func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) { - orchestratorLog.Printf("Starting workflow file parsing: %s", markdownPath) +// frontmatterParseResult holds the results of parsing and validating frontmatter +type frontmatterParseResult struct { + cleanPath string + content []byte + frontmatterResult *parser.FrontmatterResult + frontmatterForValidation map[string]any + markdownDir string + isSharedWorkflow bool +} + +// parseFrontmatterSection reads the workflow file and parses its frontmatter. +// It returns a frontmatterParseResult containing the parsed data and validation information. +// If the workflow is detected as a shared workflow (no 'on' field), isSharedWorkflow is set to true. +func (c *Compiler) parseFrontmatterSection(markdownPath string) (*frontmatterParseResult, error) { + orchestratorLog.Printf("Starting frontmatter parsing: %s", markdownPath) log.Printf("Reading file: %s", markdownPath) // Clean the path to prevent path traversal issues (gosec G304) @@ -72,12 +85,14 @@ func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) return nil, err } - // Return a special error to signal that this is a shared workflow - // and compilation should be skipped with an info message - // Note: Markdown content is optional for shared workflows (they may be just config) - return nil, &SharedWorkflowError{ - Path: cleanPath, - } + return &frontmatterParseResult{ + cleanPath: cleanPath, + content: content, + frontmatterResult: result, + frontmatterForValidation: frontmatterForValidation, + markdownDir: filepath.Dir(cleanPath), + isSharedWorkflow: true, + }, nil } // For main workflows (with 'on' field), markdown content is required @@ -99,9 +114,48 @@ func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) return nil, err } + // Validate that @include/@import directives are not used inside template regions + if err := validateNoIncludesInTemplateRegions(result.Markdown); err != nil { + orchestratorLog.Printf("Template region validation failed: %v", err) + return nil, fmt.Errorf("template region validation failed: %w", err) + } + log.Printf("Frontmatter: %d chars, Markdown: %d chars", len(result.Frontmatter), len(result.Markdown)) - markdownDir := filepath.Dir(cleanPath) + return &frontmatterParseResult{ + cleanPath: cleanPath, + content: content, + frontmatterResult: result, + frontmatterForValidation: frontmatterForValidation, + markdownDir: filepath.Dir(cleanPath), + isSharedWorkflow: false, + }, nil +} + +func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) { + orchestratorLog.Printf("Starting workflow file parsing: %s", markdownPath) + + // Parse frontmatter section + parseResult, err := c.parseFrontmatterSection(markdownPath) + if err != nil { + return nil, err + } + + // Handle shared workflows + if parseResult.isSharedWorkflow { + // Return a special error to signal that this is a shared workflow + // and compilation should be skipped with an info message + // Note: Markdown content is optional for shared workflows (they may be just config) + return nil, &SharedWorkflowError{ + Path: parseResult.cleanPath, + } + } + + // Unpack parse result for convenience + cleanPath := parseResult.cleanPath + content := parseResult.content + result := parseResult.frontmatterResult + markdownDir := parseResult.markdownDir // Extract AI engine setting from frontmatter engineSetting, engineConfig := c.ExtractEngineConfig(result.Frontmatter) @@ -151,12 +205,6 @@ func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) // This ensures strict mode doesn't leak to other workflows being compiled c.strictMode = initialStrictMode - // Validate that @include/@import directives are not used inside template regions - if err := validateNoIncludesInTemplateRegions(result.Markdown); err != nil { - orchestratorLog.Printf("Template region validation failed: %v", err) - return nil, fmt.Errorf("template region validation failed: %w", err) - } - // Override with command line AI engine setting if provided if c.engineOverride != "" { originalEngineSetting := engineSetting From eb539804586fa9bf0e9ff97942868692f206b1f0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 24 Jan 2026 05:13:34 +0000 Subject: [PATCH 3/4] Phase 2: Extract setupEngineAndImports function - Extract engine configuration, imports processing, and validation into setupEngineAndImports() - Reduces ParseWorkflowFile from 737 to 553 lines (184 line reduction, 254 total) - Creates engineSetupResult struct for intermediate data - All existing tests pass Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/compiler_orchestrator.go | 300 +++++++++++++++----------- 1 file changed, 172 insertions(+), 128 deletions(-) diff --git a/pkg/workflow/compiler_orchestrator.go b/pkg/workflow/compiler_orchestrator.go index e84625e743..d791df5b2f 100644 --- a/pkg/workflow/compiler_orchestrator.go +++ b/pkg/workflow/compiler_orchestrator.go @@ -26,136 +26,25 @@ type frontmatterParseResult struct { isSharedWorkflow bool } -// parseFrontmatterSection reads the workflow file and parses its frontmatter. -// It returns a frontmatterParseResult containing the parsed data and validation information. -// If the workflow is detected as a shared workflow (no 'on' field), isSharedWorkflow is set to true. -func (c *Compiler) parseFrontmatterSection(markdownPath string) (*frontmatterParseResult, error) { - orchestratorLog.Printf("Starting frontmatter parsing: %s", markdownPath) - log.Printf("Reading file: %s", markdownPath) - - // Clean the path to prevent path traversal issues (gosec G304) - // filepath.Clean removes ".." and other problematic path elements - cleanPath := filepath.Clean(markdownPath) - - // Read the file - content, err := os.ReadFile(cleanPath) - if err != nil { - orchestratorLog.Printf("Failed to read file: %s, error: %v", cleanPath, err) - return nil, fmt.Errorf("failed to read file: %w", err) - } - - log.Printf("File size: %d bytes", len(content)) - - // Parse frontmatter and markdown - orchestratorLog.Printf("Parsing frontmatter from file: %s", cleanPath) - result, err := parser.ExtractFrontmatterFromContent(string(content)) - if err != nil { - orchestratorLog.Printf("Frontmatter extraction failed: %v", err) - // Use FrontmatterStart from result if available, otherwise default to line 2 (after opening ---) - frontmatterStart := 2 - if result != nil && result.FrontmatterStart > 0 { - frontmatterStart = result.FrontmatterStart - } - return nil, c.createFrontmatterError(cleanPath, string(content), err, frontmatterStart) - } - - if len(result.Frontmatter) == 0 { - orchestratorLog.Print("No frontmatter found in file") - return nil, fmt.Errorf("no frontmatter found") - } - - // Preprocess schedule fields to convert human-friendly format to cron expressions - if err := c.preprocessScheduleFields(result.Frontmatter, cleanPath, string(content)); err != nil { - orchestratorLog.Printf("Schedule preprocessing failed: %v", err) - return nil, err - } - - // Create a copy of frontmatter without internal markers for schema validation - // Keep the original frontmatter with markers for YAML generation - frontmatterForValidation := c.copyFrontmatterWithoutInternalMarkers(result.Frontmatter) - - // Check if "on" field is missing - if so, treat as a shared/imported workflow - _, hasOnField := frontmatterForValidation["on"] - if !hasOnField { - detectionLog.Printf("No 'on' field detected - treating as shared agentic workflow") - - // Validate as an included/shared workflow (uses main_workflow_schema with forbidden field checks) - if err := parser.ValidateIncludedFileFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { - orchestratorLog.Printf("Shared workflow validation failed: %v", err) - return nil, err - } - - return &frontmatterParseResult{ - cleanPath: cleanPath, - content: content, - frontmatterResult: result, - frontmatterForValidation: frontmatterForValidation, - markdownDir: filepath.Dir(cleanPath), - isSharedWorkflow: true, - }, nil - } - - // For main workflows (with 'on' field), markdown content is required - if result.Markdown == "" { - orchestratorLog.Print("No markdown content found for main workflow") - return nil, fmt.Errorf("no markdown content found") - } - - // Validate main workflow frontmatter contains only expected entries - orchestratorLog.Printf("Validating main workflow frontmatter schema") - if err := parser.ValidateMainWorkflowFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { - orchestratorLog.Printf("Main workflow frontmatter validation failed: %v", err) - return nil, err - } - - // Validate event filter mutual exclusivity (branches/branches-ignore, paths/paths-ignore) - if err := ValidateEventFilters(frontmatterForValidation); err != nil { - orchestratorLog.Printf("Event filter validation failed: %v", err) - return nil, err - } - - // Validate that @include/@import directives are not used inside template regions - if err := validateNoIncludesInTemplateRegions(result.Markdown); err != nil { - orchestratorLog.Printf("Template region validation failed: %v", err) - return nil, fmt.Errorf("template region validation failed: %w", err) - } - - log.Printf("Frontmatter: %d chars, Markdown: %d chars", len(result.Frontmatter), len(result.Markdown)) - - return &frontmatterParseResult{ - cleanPath: cleanPath, - content: content, - frontmatterResult: result, - frontmatterForValidation: frontmatterForValidation, - markdownDir: filepath.Dir(cleanPath), - isSharedWorkflow: false, - }, nil +// engineSetupResult holds the results of engine configuration and validation +type engineSetupResult struct { + engineSetting string + engineConfig *EngineConfig + agenticEngine CodingAgentEngine + networkPermissions *NetworkPermissions + sandboxConfig *SandboxConfig + importsResult *parser.ImportsResult } -func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) { - orchestratorLog.Printf("Starting workflow file parsing: %s", markdownPath) - - // Parse frontmatter section - parseResult, err := c.parseFrontmatterSection(markdownPath) - if err != nil { - return nil, err - } - - // Handle shared workflows - if parseResult.isSharedWorkflow { - // Return a special error to signal that this is a shared workflow - // and compilation should be skipped with an info message - // Note: Markdown content is optional for shared workflows (they may be just config) - return nil, &SharedWorkflowError{ - Path: parseResult.cleanPath, - } - } - - // Unpack parse result for convenience - cleanPath := parseResult.cleanPath - content := parseResult.content - result := parseResult.frontmatterResult - markdownDir := parseResult.markdownDir +// setupEngineAndImports configures the AI engine, processes imports, and validates network/sandbox settings. +// This function handles: +// - Engine extraction and validation +// - Import processing and merging +// - Network permissions setup +// - Sandbox configuration +// - Strict mode validations +func (c *Compiler) setupEngineAndImports(result *parser.FrontmatterResult, cleanPath string, content []byte, markdownDir string) (*engineSetupResult, error) { + orchestratorLog.Printf("Setting up engine and processing imports") // Extract AI engine setting from frontmatter engineSetting, engineConfig := c.ExtractEngineConfig(result.Frontmatter) @@ -354,6 +243,161 @@ func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) // Restore the strict mode state after network check c.strictMode = initialStrictModeForFirewall + return &engineSetupResult{ + engineSetting: engineSetting, + engineConfig: engineConfig, + agenticEngine: agenticEngine, + networkPermissions: networkPermissions, + sandboxConfig: sandboxConfig, + importsResult: importsResult, + }, nil +} + +// parseFrontmatterSection reads the workflow file and parses its frontmatter. +// It returns a frontmatterParseResult containing the parsed data and validation information. +// If the workflow is detected as a shared workflow (no 'on' field), isSharedWorkflow is set to true. +func (c *Compiler) parseFrontmatterSection(markdownPath string) (*frontmatterParseResult, error) { + orchestratorLog.Printf("Starting frontmatter parsing: %s", markdownPath) + log.Printf("Reading file: %s", markdownPath) + + // Clean the path to prevent path traversal issues (gosec G304) + // filepath.Clean removes ".." and other problematic path elements + cleanPath := filepath.Clean(markdownPath) + + // Read the file + content, err := os.ReadFile(cleanPath) + if err != nil { + orchestratorLog.Printf("Failed to read file: %s, error: %v", cleanPath, err) + return nil, fmt.Errorf("failed to read file: %w", err) + } + + log.Printf("File size: %d bytes", len(content)) + + // Parse frontmatter and markdown + orchestratorLog.Printf("Parsing frontmatter from file: %s", cleanPath) + result, err := parser.ExtractFrontmatterFromContent(string(content)) + if err != nil { + orchestratorLog.Printf("Frontmatter extraction failed: %v", err) + // Use FrontmatterStart from result if available, otherwise default to line 2 (after opening ---) + frontmatterStart := 2 + if result != nil && result.FrontmatterStart > 0 { + frontmatterStart = result.FrontmatterStart + } + return nil, c.createFrontmatterError(cleanPath, string(content), err, frontmatterStart) + } + + if len(result.Frontmatter) == 0 { + orchestratorLog.Print("No frontmatter found in file") + return nil, fmt.Errorf("no frontmatter found") + } + + // Preprocess schedule fields to convert human-friendly format to cron expressions + if err := c.preprocessScheduleFields(result.Frontmatter, cleanPath, string(content)); err != nil { + orchestratorLog.Printf("Schedule preprocessing failed: %v", err) + return nil, err + } + + // Create a copy of frontmatter without internal markers for schema validation + // Keep the original frontmatter with markers for YAML generation + frontmatterForValidation := c.copyFrontmatterWithoutInternalMarkers(result.Frontmatter) + + // Check if "on" field is missing - if so, treat as a shared/imported workflow + _, hasOnField := frontmatterForValidation["on"] + if !hasOnField { + detectionLog.Printf("No 'on' field detected - treating as shared agentic workflow") + + // Validate as an included/shared workflow (uses main_workflow_schema with forbidden field checks) + if err := parser.ValidateIncludedFileFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { + orchestratorLog.Printf("Shared workflow validation failed: %v", err) + return nil, err + } + + return &frontmatterParseResult{ + cleanPath: cleanPath, + content: content, + frontmatterResult: result, + frontmatterForValidation: frontmatterForValidation, + markdownDir: filepath.Dir(cleanPath), + isSharedWorkflow: true, + }, nil + } + + // For main workflows (with 'on' field), markdown content is required + if result.Markdown == "" { + orchestratorLog.Print("No markdown content found for main workflow") + return nil, fmt.Errorf("no markdown content found") + } + + // Validate main workflow frontmatter contains only expected entries + orchestratorLog.Printf("Validating main workflow frontmatter schema") + if err := parser.ValidateMainWorkflowFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { + orchestratorLog.Printf("Main workflow frontmatter validation failed: %v", err) + return nil, err + } + + // Validate event filter mutual exclusivity (branches/branches-ignore, paths/paths-ignore) + if err := ValidateEventFilters(frontmatterForValidation); err != nil { + orchestratorLog.Printf("Event filter validation failed: %v", err) + return nil, err + } + + // Validate that @include/@import directives are not used inside template regions + if err := validateNoIncludesInTemplateRegions(result.Markdown); err != nil { + orchestratorLog.Printf("Template region validation failed: %v", err) + return nil, fmt.Errorf("template region validation failed: %w", err) + } + + log.Printf("Frontmatter: %d chars, Markdown: %d chars", len(result.Frontmatter), len(result.Markdown)) + + return &frontmatterParseResult{ + cleanPath: cleanPath, + content: content, + frontmatterResult: result, + frontmatterForValidation: frontmatterForValidation, + markdownDir: filepath.Dir(cleanPath), + isSharedWorkflow: false, + }, nil +} + +func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) { + orchestratorLog.Printf("Starting workflow file parsing: %s", markdownPath) + + // Parse frontmatter section + parseResult, err := c.parseFrontmatterSection(markdownPath) + if err != nil { + return nil, err + } + + // Handle shared workflows + if parseResult.isSharedWorkflow { + // Return a special error to signal that this is a shared workflow + // and compilation should be skipped with an info message + // Note: Markdown content is optional for shared workflows (they may be just config) + return nil, &SharedWorkflowError{ + Path: parseResult.cleanPath, + } + } + + // Unpack parse result for convenience + cleanPath := parseResult.cleanPath + content := parseResult.content + result := parseResult.frontmatterResult + markdownDir := parseResult.markdownDir + + // Setup engine and process imports + engineSetup, err := c.setupEngineAndImports(result, cleanPath, content, markdownDir) + if err != nil { + return nil, err + } + + // Unpack engine setup results + engineSetting := engineSetup.engineSetting + engineConfig := engineSetup.engineConfig + agenticEngine := engineSetup.agenticEngine + networkPermissions := engineSetup.networkPermissions + sandboxConfig := engineSetup.sandboxConfig + importsResult := engineSetup.importsResult + log.Print("Processing tools and includes...") // Extract SafeOutputs configuration early so we can use it when applying default tools From a72e9279ab9fe18af012021c01e23f001d9bb8e0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 24 Jan 2026 05:17:33 +0000 Subject: [PATCH 4/4] Phase 3: Extract processToolsAndMarkdown function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract tools, runtimes, and markdown processing into processToolsAndMarkdown() - Reduces ParseWorkflowFile from 553 to 383 lines (170 line reduction, 424 total) - Creates toolsProcessingResult struct for intermediate data - All existing tests pass - Total reduction: 52.5% from original (807 → 383 lines) Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/compiler_orchestrator.go | 438 +++++++++++++++----------- 1 file changed, 252 insertions(+), 186 deletions(-) diff --git a/pkg/workflow/compiler_orchestrator.go b/pkg/workflow/compiler_orchestrator.go index d791df5b2f..330a8b8c6d 100644 --- a/pkg/workflow/compiler_orchestrator.go +++ b/pkg/workflow/compiler_orchestrator.go @@ -36,6 +36,242 @@ type engineSetupResult struct { importsResult *parser.ImportsResult } +// toolsProcessingResult holds the results of tools and markdown processing +type toolsProcessingResult struct { + tools map[string]any + runtimes map[string]any + toolsTimeout int + toolsStartupTimeout int + markdownContent string + allIncludedFiles []string + workflowName string + frontmatterName string + needsTextOutput bool + trackerID string + safeOutputs *SafeOutputsConfig + secretMasking *SecretMaskingConfig + parsedFrontmatter *FrontmatterConfig +} + +// processToolsAndMarkdown processes tools configuration, runtimes, and markdown content. +// This function handles: +// - Safe outputs and secret masking configuration +// - Tools and MCP servers merging +// - Runtimes merging +// - MCP validations +// - Markdown content expansion +// - Workflow name extraction +func (c *Compiler) processToolsAndMarkdown(result *parser.FrontmatterResult, cleanPath string, markdownDir string, + agenticEngine CodingAgentEngine, engineSetting string, importsResult *parser.ImportsResult) (*toolsProcessingResult, error) { + + orchestratorLog.Printf("Processing tools and markdown") + log.Print("Processing tools and includes...") + + // Extract SafeOutputs configuration early so we can use it when applying default tools + safeOutputs := c.extractSafeOutputsConfig(result.Frontmatter) + + // Extract SecretMasking configuration + secretMasking := c.extractSecretMaskingConfig(result.Frontmatter) + + // Merge secret-masking from imports with top-level secret-masking + if importsResult.MergedSecretMasking != "" { + orchestratorLog.Printf("Merging secret-masking from imports") + var err error + secretMasking, err = c.MergeSecretMasking(secretMasking, importsResult.MergedSecretMasking) + if err != nil { + orchestratorLog.Printf("Secret-masking merge failed: %v", err) + return nil, fmt.Errorf("failed to merge secret-masking: %w", err) + } + } + + var tools map[string]any + + // Extract tools from the main file + topTools := extractToolsFromFrontmatter(result.Frontmatter) + + // Extract mcp-servers from the main file and merge them into tools + mcpServers := extractMCPServersFromFrontmatter(result.Frontmatter) + + // Process @include directives to extract additional tools + orchestratorLog.Printf("Expanding includes for tools") + includedTools, includedToolFiles, err := parser.ExpandIncludesWithManifest(result.Markdown, markdownDir, true) + if err != nil { + orchestratorLog.Printf("Failed to expand includes for tools: %v", err) + return nil, fmt.Errorf("failed to expand includes for tools: %w", err) + } + + // Combine imported tools with included tools + var allIncludedTools string + if importsResult.MergedTools != "" && includedTools != "" { + allIncludedTools = importsResult.MergedTools + "\n" + includedTools + } else if importsResult.MergedTools != "" { + allIncludedTools = importsResult.MergedTools + } else { + allIncludedTools = includedTools + } + + // Combine imported mcp-servers with top-level mcp-servers + // Imported mcp-servers are in JSON format (newline-separated), need to merge them + allMCPServers := mcpServers + if importsResult.MergedMCPServers != "" { + orchestratorLog.Printf("Merging imported mcp-servers") + // Parse and merge imported MCP servers + mergedMCPServers, err := c.MergeMCPServers(mcpServers, importsResult.MergedMCPServers) + if err != nil { + orchestratorLog.Printf("MCP servers merge failed: %v", err) + return nil, fmt.Errorf("failed to merge imported mcp-servers: %w", err) + } + allMCPServers = mergedMCPServers + } + + // Merge tools including mcp-servers + orchestratorLog.Printf("Merging tools and MCP servers") + tools, err = c.mergeToolsAndMCPServers(topTools, allMCPServers, allIncludedTools) + if err != nil { + orchestratorLog.Printf("Tools merge failed: %v", err) + return nil, fmt.Errorf("failed to merge tools: %w", err) + } + + // Extract and validate tools timeout settings + toolsTimeout, err := c.extractToolsTimeout(tools) + if err != nil { + return nil, fmt.Errorf("invalid tools timeout configuration: %w", err) + } + + toolsStartupTimeout, err := c.extractToolsStartupTimeout(tools) + if err != nil { + return nil, fmt.Errorf("invalid tools startup timeout configuration: %w", err) + } + + // Remove meta fields (timeout, startup-timeout) from merged tools map + // These are configuration fields, not actual tools + delete(tools, "timeout") + delete(tools, "startup-timeout") + + // Extract and merge runtimes from frontmatter and imports + topRuntimes := extractRuntimesFromFrontmatter(result.Frontmatter) + orchestratorLog.Printf("Merging runtimes") + runtimes, err := mergeRuntimes(topRuntimes, importsResult.MergedRuntimes) + if err != nil { + orchestratorLog.Printf("Runtimes merge failed: %v", err) + return nil, fmt.Errorf("failed to merge runtimes: %w", err) + } + + // Add MCP fetch server if needed (when web-fetch is requested but engine doesn't support it) + tools, _ = AddMCPFetchServerIfNeeded(tools, agenticEngine) + + // Validate MCP configurations + orchestratorLog.Printf("Validating MCP configurations") + if err := ValidateMCPConfigs(tools); err != nil { + orchestratorLog.Printf("MCP configuration validation failed: %v", err) + return nil, err + } + + // Validate HTTP transport support for the current engine + if err := c.validateHTTPTransportSupport(tools, agenticEngine); err != nil { + orchestratorLog.Printf("HTTP transport validation failed: %v", err) + return nil, err + } + + if !agenticEngine.SupportsToolsAllowlist() { + // For engines that don't support tool allowlists (like codex), ignore tools section and provide warnings + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Using experimental %s support (engine: %s)", agenticEngine.GetDisplayName(), engineSetting))) + c.IncrementWarningCount() + if _, hasTools := result.Frontmatter["tools"]; hasTools { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("'tools' section ignored when using engine: %s (%s doesn't support MCP tool allow-listing)", engineSetting, agenticEngine.GetDisplayName()))) + c.IncrementWarningCount() + } + tools = map[string]any{} + // For now, we'll add a basic github tool (always uses docker MCP) + githubConfig := map[string]any{} + tools["github"] = githubConfig + } + + // Validate max-turns support for the current engine + if err := c.validateMaxTurnsSupport(result.Frontmatter, agenticEngine); err != nil { + return nil, err + } + + // Validate web-search support for the current engine (warning only) + c.validateWebSearchSupport(tools, agenticEngine) + + // Process @include directives in markdown content + markdownContent, includedMarkdownFiles, err := parser.ExpandIncludesWithManifest(result.Markdown, markdownDir, false) + if err != nil { + return nil, fmt.Errorf("failed to expand includes in markdown: %w", err) + } + + // Prepend imported markdown from frontmatter imports field + if importsResult.MergedMarkdown != "" { + markdownContent = importsResult.MergedMarkdown + markdownContent + } + + log.Print("Expanded includes in markdown content") + + // Combine all included files (from tools and markdown) + // Use a map to deduplicate files + allIncludedFilesMap := make(map[string]bool) + for _, file := range includedToolFiles { + allIncludedFilesMap[file] = true + } + for _, file := range includedMarkdownFiles { + allIncludedFilesMap[file] = true + } + var allIncludedFiles []string + for file := range allIncludedFilesMap { + allIncludedFiles = append(allIncludedFiles, file) + } + // Sort files alphabetically to ensure consistent ordering in lock files + sort.Strings(allIncludedFiles) + + // Extract workflow name + workflowName, err := parser.ExtractWorkflowNameFromMarkdown(cleanPath) + if err != nil { + return nil, fmt.Errorf("failed to extract workflow name: %w", err) + } + + // Check if frontmatter specifies a custom name and use it instead + frontmatterName := extractStringFromMap(result.Frontmatter, "name", nil) + if frontmatterName != "" { + workflowName = frontmatterName + } + + log.Printf("Extracted workflow name: '%s'", workflowName) + + // Check if the markdown content uses the text output + needsTextOutput := c.detectTextOutputUsage(markdownContent) + + // Extract and validate tracker-id + trackerID, err := c.extractTrackerID(result.Frontmatter) + if err != nil { + return nil, err + } + + // Parse frontmatter config once for performance optimization + parsedFrontmatter, err := ParseFrontmatterConfig(result.Frontmatter) + if err != nil { + orchestratorLog.Printf("Failed to parse frontmatter config: %v", err) + // Non-fatal error - continue with nil ParsedFrontmatter + parsedFrontmatter = nil + } + + return &toolsProcessingResult{ + tools: tools, + runtimes: runtimes, + toolsTimeout: toolsTimeout, + toolsStartupTimeout: toolsStartupTimeout, + markdownContent: markdownContent, + allIncludedFiles: allIncludedFiles, + workflowName: workflowName, + frontmatterName: frontmatterName, + needsTextOutput: needsTextOutput, + trackerID: trackerID, + safeOutputs: safeOutputs, + secretMasking: secretMasking, + parsedFrontmatter: parsedFrontmatter, + }, nil +} + // setupEngineAndImports configures the AI engine, processes imports, and validates network/sandbox settings. // This function handles: // - Engine extraction and validation @@ -398,196 +634,26 @@ func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) sandboxConfig := engineSetup.sandboxConfig importsResult := engineSetup.importsResult - log.Print("Processing tools and includes...") - - // Extract SafeOutputs configuration early so we can use it when applying default tools - safeOutputs := c.extractSafeOutputsConfig(result.Frontmatter) - - // Extract SecretMasking configuration - secretMasking := c.extractSecretMaskingConfig(result.Frontmatter) - - // Merge secret-masking from imports with top-level secret-masking - if importsResult.MergedSecretMasking != "" { - orchestratorLog.Printf("Merging secret-masking from imports") - secretMasking, err = c.MergeSecretMasking(secretMasking, importsResult.MergedSecretMasking) - if err != nil { - orchestratorLog.Printf("Secret-masking merge failed: %v", err) - return nil, fmt.Errorf("failed to merge secret-masking: %w", err) - } - } - - var tools map[string]any - - // Extract tools from the main file - topTools := extractToolsFromFrontmatter(result.Frontmatter) - - // Extract mcp-servers from the main file and merge them into tools - mcpServers := extractMCPServersFromFrontmatter(result.Frontmatter) - - // Process @include directives to extract additional tools - orchestratorLog.Printf("Expanding includes for tools") - includedTools, includedToolFiles, err := parser.ExpandIncludesWithManifest(result.Markdown, markdownDir, true) - if err != nil { - orchestratorLog.Printf("Failed to expand includes for tools: %v", err) - return nil, fmt.Errorf("failed to expand includes for tools: %w", err) - } - - // Combine imported tools with included tools - var allIncludedTools string - if importsResult.MergedTools != "" && includedTools != "" { - allIncludedTools = importsResult.MergedTools + "\n" + includedTools - } else if importsResult.MergedTools != "" { - allIncludedTools = importsResult.MergedTools - } else { - allIncludedTools = includedTools - } - - // Combine imported mcp-servers with top-level mcp-servers - // Imported mcp-servers are in JSON format (newline-separated), need to merge them - allMCPServers := mcpServers - if importsResult.MergedMCPServers != "" { - orchestratorLog.Printf("Merging imported mcp-servers") - // Parse and merge imported MCP servers - mergedMCPServers, err := c.MergeMCPServers(mcpServers, importsResult.MergedMCPServers) - if err != nil { - orchestratorLog.Printf("MCP servers merge failed: %v", err) - return nil, fmt.Errorf("failed to merge imported mcp-servers: %w", err) - } - allMCPServers = mergedMCPServers - } - - // Merge tools including mcp-servers - orchestratorLog.Printf("Merging tools and MCP servers") - tools, err = c.mergeToolsAndMCPServers(topTools, allMCPServers, allIncludedTools) - - if err != nil { - orchestratorLog.Printf("Tools merge failed: %v", err) - return nil, fmt.Errorf("failed to merge tools: %w", err) - } - - // Extract and validate tools timeout settings - toolsTimeout, err := c.extractToolsTimeout(tools) - if err != nil { - return nil, fmt.Errorf("invalid tools timeout configuration: %w", err) - } - - toolsStartupTimeout, err := c.extractToolsStartupTimeout(tools) - if err != nil { - return nil, fmt.Errorf("invalid tools startup timeout configuration: %w", err) - } - - // Remove meta fields (timeout, startup-timeout) from merged tools map - // These are configuration fields, not actual tools - delete(tools, "timeout") - delete(tools, "startup-timeout") - - // Extract and merge runtimes from frontmatter and imports - topRuntimes := extractRuntimesFromFrontmatter(result.Frontmatter) - orchestratorLog.Printf("Merging runtimes") - runtimes, err := mergeRuntimes(topRuntimes, importsResult.MergedRuntimes) - if err != nil { - orchestratorLog.Printf("Runtimes merge failed: %v", err) - return nil, fmt.Errorf("failed to merge runtimes: %w", err) - } - - // Add MCP fetch server if needed (when web-fetch is requested but engine doesn't support it) - tools, _ = AddMCPFetchServerIfNeeded(tools, agenticEngine) - - // Validate MCP configurations - orchestratorLog.Printf("Validating MCP configurations") - if err := ValidateMCPConfigs(tools); err != nil { - orchestratorLog.Printf("MCP configuration validation failed: %v", err) - return nil, err - } - - // Validate HTTP transport support for the current engine - if err := c.validateHTTPTransportSupport(tools, agenticEngine); err != nil { - orchestratorLog.Printf("HTTP transport validation failed: %v", err) - return nil, err - } - - if !agenticEngine.SupportsToolsAllowlist() { - // For engines that don't support tool allowlists (like codex), ignore tools section and provide warnings - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Using experimental %s support (engine: %s)", agenticEngine.GetDisplayName(), engineSetting))) - c.IncrementWarningCount() - if _, hasTools := result.Frontmatter["tools"]; hasTools { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("'tools' section ignored when using engine: %s (%s doesn't support MCP tool allow-listing)", engineSetting, agenticEngine.GetDisplayName()))) - c.IncrementWarningCount() - } - tools = map[string]any{} - // For now, we'll add a basic github tool (always uses docker MCP) - githubConfig := map[string]any{} - - tools["github"] = githubConfig - } - - // Validate max-turns support for the current engine - if err := c.validateMaxTurnsSupport(result.Frontmatter, agenticEngine); err != nil { - return nil, err - } - - // Validate web-search support for the current engine (warning only) - c.validateWebSearchSupport(tools, agenticEngine) - - // Process @include directives in markdown content - markdownContent, includedMarkdownFiles, err := parser.ExpandIncludesWithManifest(result.Markdown, markdownDir, false) - if err != nil { - return nil, fmt.Errorf("failed to expand includes in markdown: %w", err) - } - - // Prepend imported markdown from frontmatter imports field - if importsResult.MergedMarkdown != "" { - markdownContent = importsResult.MergedMarkdown + markdownContent - } - - log.Print("Expanded includes in markdown content") - - // Combine all included files (from tools and markdown) - // Use a map to deduplicate files - allIncludedFilesMap := make(map[string]bool) - for _, file := range includedToolFiles { - allIncludedFilesMap[file] = true - } - for _, file := range includedMarkdownFiles { - allIncludedFilesMap[file] = true - } - var allIncludedFiles []string - for file := range allIncludedFilesMap { - allIncludedFiles = append(allIncludedFiles, file) - } - // Sort files alphabetically to ensure consistent ordering in lock files - sort.Strings(allIncludedFiles) - - // Extract workflow name - workflowName, err := parser.ExtractWorkflowNameFromMarkdown(cleanPath) - if err != nil { - return nil, fmt.Errorf("failed to extract workflow name: %w", err) - } - - // Check if frontmatter specifies a custom name and use it instead - frontmatterName := extractStringFromMap(result.Frontmatter, "name", nil) - if frontmatterName != "" { - workflowName = frontmatterName - } - - log.Printf("Extracted workflow name: '%s'", workflowName) - - // Check if the markdown content uses the text output - needsTextOutput := c.detectTextOutputUsage(markdownContent) - - // Extract and validate tracker-id - trackerID, err := c.extractTrackerID(result.Frontmatter) + // Process tools and markdown + toolsResult, err := c.processToolsAndMarkdown(result, cleanPath, markdownDir, agenticEngine, engineSetting, importsResult) if err != nil { return nil, err } - // Parse frontmatter config once for performance optimization - parsedFrontmatter, err := ParseFrontmatterConfig(result.Frontmatter) - if err != nil { - orchestratorLog.Printf("Failed to parse frontmatter config: %v", err) - // Non-fatal error - continue with nil ParsedFrontmatter - parsedFrontmatter = nil - } + // Unpack tools processing results + tools := toolsResult.tools + runtimes := toolsResult.runtimes + toolsTimeout := toolsResult.toolsTimeout + toolsStartupTimeout := toolsResult.toolsStartupTimeout + markdownContent := toolsResult.markdownContent + allIncludedFiles := toolsResult.allIncludedFiles + workflowName := toolsResult.workflowName + frontmatterName := toolsResult.frontmatterName + needsTextOutput := toolsResult.needsTextOutput + trackerID := toolsResult.trackerID + safeOutputs := toolsResult.safeOutputs + secretMasking := toolsResult.secretMasking + parsedFrontmatter := toolsResult.parsedFrontmatter // Build workflow data workflowData := &WorkflowData{