Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
6097248
Initial plan
Copilot Mar 11, 2026
8a098cc
feat: replace inlined Go builtin engine definitions with embedded YAM…
Copilot Mar 11, 2026
4a6f53a
fix: add missing auth bindings, engine key wrapper, and schema valida…
Copilot Mar 11, 2026
2eee3ff
fix: allow inline engine definition (engine.runtime) to be imported f…
Copilot Mar 11, 2026
aa1af02
feat: convert engine yml files to shared agentic workflow md files wi…
Copilot Mar 11, 2026
5ae6466
fix: xml-comment engine md body to avoid polluting prompt; fix CLI en…
Copilot Mar 11, 2026
ef76fa7
Update pkg/workflow/engine_definition.go
pelikhan Mar 11, 2026
63714b4
Merge branch 'main' into copilot/replace-inlined-engines-with-yml-res…
pelikhan Mar 11, 2026
e5e57b5
fix: address automated review comments (virtual FS safety, CRLF front…
Copilot Mar 11, 2026
889894c
fix: validate builtin engine files against shared workflow schema
Copilot Mar 11, 2026
48a4ecc
fix: remove engine_definition_schema.json, validation via main workfl…
Copilot Mar 11, 2026
b3ae08b
Merge branch 'main' into copilot/replace-inlined-engines-with-yml-res…
pelikhan Mar 11, 2026
2ea2345
Merge branch 'main' into copilot/replace-inlined-engines-with-yml-res…
pelikhan Mar 11, 2026
f3dfe5a
fix: resolve lint errors and update test fixtures for action version …
Copilot Mar 11, 2026
81cc602
Merge branch 'main' into copilot/replace-inlined-engines-with-yml-res…
pelikhan Mar 11, 2026
c536da1
Add changeset [skip-ci]
github-actions[bot] Mar 11, 2026
f5e3621
Merge branch 'main' into copilot/replace-inlined-engines-with-yml-res…
pelikhan Mar 11, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/patch-embed-engine-markdown.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 5 additions & 3 deletions pkg/parser/import_field_extractor.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,13 +78,15 @@ func (acc *importAccumulator) extractAllImportFields(content []byte, item import

// Track import path for runtime-import macro generation (only if no inputs).
// Imports with inputs must be inlined for compile-time substitution.
// Builtin paths (@builtin:…) are pure configuration — they carry no user-visible
// prompt content and must not generate runtime-import macros.
importRelPath := computeImportRelPath(item.fullPath, item.importPath)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good guard — combining the inputs check with the BuiltinPathPrefix check in a single condition keeps the logic concise. The comment on lines 80-81 makes the intent clear for future readers.

if len(item.inputs) == 0 {
// No inputs - use runtime-import macro
if len(item.inputs) == 0 && !strings.HasPrefix(importRelPath, BuiltinPathPrefix) {
// No inputs and not a builtin - use runtime-import macro
acc.importPaths = append(acc.importPaths, importRelPath)
log.Printf("Added import path for runtime-import: %s", importRelPath)
} else {
} else if len(item.inputs) > 0 {
// Has inputs - must inline for compile-time substitution
log.Printf("Import %s has inputs - will be inlined for compile-time substitution", importRelPath)
markdownContent, err := processIncludedFileWithVisited(item.fullPath, item.sectionName, false, visited)
Expand Down
10 changes: 10 additions & 0 deletions pkg/parser/remote_fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,16 @@ func isRepositoryImport(importPath string) bool {
func ResolveIncludePath(filePath, baseDir string, cache *ImportCache) (string, error) {
remoteLog.Printf("Resolving include path: file_path=%s, base_dir=%s", filePath, baseDir)

// Handle builtin paths - these are embedded files that bypass filesystem resolution.
// No security check is needed since the content is compiled into the binary.
if strings.HasPrefix(filePath, BuiltinPathPrefix) {
if !BuiltinVirtualFileExists(filePath) {
return "", fmt.Errorf("builtin file not found: %s", filePath)
}
remoteLog.Printf("Resolved builtin path: %s", filePath)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice early-return pattern for builtin paths. Bypassing the filesystem resolution for embedded files makes sense, and the BuiltinVirtualFileExists check provides a helpful error for typos in builtin path references.

return filePath, nil
}

// Check if this is a workflowspec (contains owner/repo/path format)
// Format: owner/repo/path@ref or owner/repo/path@ref#section
if isWorkflowSpec(filePath) {
Expand Down
8 changes: 8 additions & 0 deletions pkg/parser/remote_fetch_wasm.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,14 @@ func isRepositoryImport(importPath string) bool {
}

func ResolveIncludePath(filePath, baseDir string, cache *ImportCache) (string, error) {
// Handle builtin paths - these are embedded files that bypass filesystem resolution.
if strings.HasPrefix(filePath, BuiltinPathPrefix) {
if !BuiltinVirtualFileExists(filePath) {
return "", fmt.Errorf("builtin file not found: %s", filePath)
}
return filePath, nil
}

if isWorkflowSpec(filePath) {
return "", fmt.Errorf("remote imports not available in Wasm: %s", filePath)
}
Expand Down
132 changes: 132 additions & 0 deletions pkg/parser/schemas/main_workflow_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -8076,6 +8076,138 @@
},
"required": ["runtime"],
"additionalProperties": false
},
{
"type": "object",
"description": "Engine definition: full declarative metadata for a named engine entry (used in builtin engine shared workflow files such as @builtin:engines/*.md)",
"properties": {
"id": {
"type": "string",
"description": "Unique engine identifier (e.g. 'copilot', 'claude', 'codex', 'gemini')"
},
"display-name": {
"type": "string",
"description": "Human-readable display name for the engine"
},
"description": {
"type": "string",
"description": "Human-readable description of the engine"
},
"runtime-id": {
"type": "string",
"description": "Runtime adapter identifier. Maps to the CodingAgentEngine registered in the engine registry. Defaults to id when omitted."
},
"provider": {
"type": "object",
"description": "Provider metadata for the engine",
"properties": {
"name": {
"type": "string",
"description": "Provider name (e.g. 'anthropic', 'github', 'google', 'openai')"
},
"auth": {
"type": "object",
"description": "Default authentication configuration for the provider",
"properties": {
"secret": {
"type": "string",
"description": "Name of the GitHub Actions secret that contains the API key"
},
"strategy": {
"type": "string",
"enum": ["api-key", "oauth-client-credentials", "bearer"],
"description": "Authentication strategy"
},
"token-url": {
"type": "string",
"description": "OAuth 2.0 token endpoint URL"
},
"client-id": {
"type": "string",
"description": "GitHub Actions secret name for the OAuth client ID"
},
"client-secret": {
"type": "string",
"description": "GitHub Actions secret name for the OAuth client secret"
},
"token-field": {
"type": "string",
"description": "JSON field name in the token response containing the access token"
},
"header-name": {
"type": "string",
"description": "HTTP header name to inject the API key or token into"
}
},
"additionalProperties": false
},
"request": {
"type": "object",
"description": "Request shaping configuration",
"properties": {
"path-template": {
"type": "string",
"description": "URL path template with variable placeholders"
},
"query": {
"type": "object",
"description": "Static query parameters",
"additionalProperties": { "type": "string" }
},
"body-inject": {
"type": "object",
"description": "Key/value pairs injected into the JSON request body",
"additionalProperties": { "type": "string" }
}
},
"additionalProperties": false
}
},
"additionalProperties": false
},
"models": {
"type": "object",
"description": "Model selection configuration for the engine",
"properties": {
"default": {
"type": "string",
"description": "Default model identifier"
},
"supported": {
"type": "array",
"items": { "type": "string" },
"description": "List of supported model identifiers"
}
},
"additionalProperties": false
},
"auth": {
"type": "array",
"description": "Authentication bindings — maps logical roles (e.g. 'api-key') to GitHub Actions secret names",
"items": {
"type": "object",
"properties": {
"role": {
"type": "string",
"description": "Logical authentication role (e.g. 'api-key', 'token')"
},
"secret": {
"type": "string",
"description": "Name of the GitHub Actions secret that provides credentials for this role"
}
},
"required": ["role", "secret"],
"additionalProperties": false
}
},
"options": {
"type": "object",
"description": "Additional engine-specific options",
"additionalProperties": true
}
},
"required": ["id", "display-name"],
"additionalProperties": false
}
]
},
Expand Down
61 changes: 59 additions & 2 deletions pkg/parser/virtual_fs.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,68 @@
package parser

import "os"
import (
"fmt"
"os"
"strings"
"sync"
)

// builtinVirtualFiles holds embedded built-in files registered at startup.
// Keys use the "@builtin:" path prefix (e.g. "@builtin:engines/copilot.md").
// The map is populated once and then read-only; concurrent reads are safe.
var (
builtinVirtualFiles map[string][]byte
builtinVirtualFilesMu sync.RWMutex
)

// RegisterBuiltinVirtualFile registers an embedded file under a canonical builtin path.
// Paths must start with BuiltinPathPrefix ("@builtin:"); it panics if they do not.
// If the same path is registered twice with identical content the call is a no-op.
// Registering the same path with different content panics to surface configuration errors early.
// This function is safe for concurrent use.
func RegisterBuiltinVirtualFile(path string, content []byte) {
if !strings.HasPrefix(path, BuiltinPathPrefix) {
panic(fmt.Sprintf("RegisterBuiltinVirtualFile: path %q does not start with %q", path, BuiltinPathPrefix))
}
builtinVirtualFilesMu.Lock()
defer builtinVirtualFilesMu.Unlock()
if builtinVirtualFiles == nil {
builtinVirtualFiles = make(map[string][]byte)
}
if existing, ok := builtinVirtualFiles[path]; ok {
if string(existing) != string(content) {
panic(fmt.Sprintf("RegisterBuiltinVirtualFile: path %q already registered with different content", path))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Panicking on duplicate registration with different content is a good safety net during startup, but consider adding a note about the registration order. If two init() functions race to register the same builtin path with different content (e.g., during test parallelism), the panic message alone might not reveal which caller "won". A logging statement before the panic with caller info could help diagnose such issues.

}
return // idempotent: same content, no-op
}
builtinVirtualFiles[path] = content
}

// BuiltinVirtualFileExists returns true if the given path is registered as a builtin virtual file.
func BuiltinVirtualFileExists(path string) bool {
builtinVirtualFilesMu.RLock()
defer builtinVirtualFilesMu.RUnlock()
_, ok := builtinVirtualFiles[path]
return ok
}

// BuiltinPathPrefix is the path prefix used for embedded builtin files.
// Paths with this prefix bypass filesystem resolution and security checks.
const BuiltinPathPrefix = "@builtin:"

// readFileFunc is the function used to read file contents throughout the parser.
// In wasm builds, this is overridden to read from a virtual filesystem
// populated by the browser via SetVirtualFiles.
var readFileFunc = os.ReadFile
// In native builds, builtin virtual files are checked first, then os.ReadFile.
var readFileFunc = func(path string) ([]byte, error) {
builtinVirtualFilesMu.RLock()
content, ok := builtinVirtualFiles[path]
builtinVirtualFilesMu.RUnlock()
if ok {
return content, nil
}
return os.ReadFile(path)
}

// ReadFile reads a file using the parser's file reading function, which
// checks the virtual filesystem first in wasm builds. Use this instead of
Expand Down
7 changes: 7 additions & 0 deletions pkg/parser/virtual_fs_wasm.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@ func VirtualFileExists(path string) bool {
func init() {
// Override readFileFunc in wasm builds to check virtual files first.
readFileFunc = func(path string) ([]byte, error) {
// Check builtin virtual files first (embedded engine .md files etc.)
builtinVirtualFilesMu.RLock()
builtinContent, builtinOK := builtinVirtualFiles[path]
builtinVirtualFilesMu.RUnlock()
if builtinOK {
return builtinContent, nil
}
if virtualFiles != nil {
if content, ok := virtualFiles[path]; ok {
return content, nil
Expand Down
74 changes: 74 additions & 0 deletions pkg/workflow/compiler_orchestrator_engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,27 @@ func (c *Compiler) setupEngineAndImports(result *parser.FrontmatterResult, clean
c.IncrementWarningCount()
}
engineSetting = c.engineOverride
// Update engineConfig.ID so that downstream code (e.g. generateCreateAwInfo) uses
// the override engine ID, not the one parsed from the frontmatter.
if engineConfig != nil {
engineConfig.ID = c.engineOverride
}
}

// When the engine is specified in short/string form ("engine: copilot") and no CLI
// override is active, inject the corresponding builtin shared-workflow .md as an
// import. This makes "engine: copilot" syntactic sugar for importing the builtin
// copilot.md, which carries the full engine definition. The engine field is removed
// from the frontmatter so the definition comes entirely from the import.
if c.engineOverride == "" && isStringFormEngine(result.Frontmatter) && engineSetting != "" {
builtinPath := builtinEnginePath(engineSetting)
if parser.BuiltinVirtualFileExists(builtinPath) {
orchestratorEngineLog.Printf("Injecting builtin engine import: %s", builtinPath)
addImportToFrontmatter(result.Frontmatter, builtinPath)
delete(result.Frontmatter, "engine")
engineSetting = ""
engineConfig = nil
}
}

// Process imports from frontmatter first (before @include directives)
Expand Down Expand Up @@ -197,6 +218,19 @@ func (c *Compiler) setupEngineAndImports(result *parser.FrontmatterResult, clean
return nil, fmt.Errorf("failed to extract engine config from included file: %w", err)
}
engineConfig = extractedConfig

// If the imported engine is an inline definition (engine.runtime sub-object),
// validate and register it in the catalog. This mirrors the handling for inline
// definitions declared directly in the main workflow (above).
if engineConfig != nil && engineConfig.IsInlineDefinition {
if err := c.validateEngineInlineDefinition(engineConfig); err != nil {
return nil, err
}
if err := c.validateEngineAuthDefinition(engineConfig); err != nil {
return nil, err
}
c.registerInlineEngineDefinition(engineConfig)
}
}

// Apply the default AI engine setting if not specified
Expand Down Expand Up @@ -299,3 +333,43 @@ func (c *Compiler) setupEngineAndImports(result *parser.FrontmatterResult, clean
configSteps: configSteps,
}, nil
}

// isStringFormEngine reports whether the "engine" field in the given frontmatter is a
// plain string (e.g. "engine: copilot"), as opposed to an object with an "id" or
// "runtime" sub-key.
func isStringFormEngine(frontmatter map[string]any) bool {
engine, exists := frontmatter["engine"]
if !exists {
return false
}
_, isString := engine.(string)
return isString
}

// addImportToFrontmatter appends importPath to the "imports" slice in frontmatter.
// It handles the case where "imports" may be absent, a []any, a []string, or a
// single string (which is converted to a two-element slice preserving the original value).
// Any other unexpected type is left unchanged and importPath is not injected.
func addImportToFrontmatter(frontmatter map[string]any, importPath string) {
existing, hasImports := frontmatter["imports"]
if !hasImports {
frontmatter["imports"] = []any{importPath}
return
}
switch v := existing.(type) {
case []any:
frontmatter["imports"] = append(v, importPath)
case []string:
newSlice := make([]any, len(v)+1)
for i, s := range v {
newSlice[i] = s
}
newSlice[len(v)] = importPath
frontmatter["imports"] = newSlice
case string:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The case string branch correctly handles the single-string-to-slice promotion. The comment on the default case (inside case string:) appears slightly misplaced — the comment "For any other unexpected type, leave the field untouched..." actually describes a scenario handled after the switch, but it's placed inside the case string: block. Consider moving it outside the switch to avoid confusion for future readers.

// Single string import — preserve it and append the new one.
frontmatter["imports"] = []any{v, importPath}
// For any other unexpected type, leave the field untouched so the
// downstream parser can still report its own error for the invalid value.
}
}
15 changes: 12 additions & 3 deletions pkg/workflow/compiler_yaml.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,22 @@ func (c *Compiler) generateWorkflowHeader(yaml *strings.Builder, data *WorkflowD
}

// Add manifest of imported/included files if any exist
if len(data.ImportedFiles) > 0 || len(data.IncludedFiles) > 0 {
// Build a user-visible imports list by filtering out internal builtin engine paths
// (e.g. "@builtin:engines/copilot.md") which are implementation details.
var visibleImports []string
for _, file := range data.ImportedFiles {
if !strings.HasPrefix(file, parser.BuiltinPathPrefix) {
visibleImports = append(visibleImports, file)
}
}

if len(visibleImports) > 0 || len(data.IncludedFiles) > 0 {
yaml.WriteString("#\n")
yaml.WriteString("# Resolved workflow manifest:\n")

if len(data.ImportedFiles) > 0 {
if len(visibleImports) > 0 {
yaml.WriteString("# Imports:\n")
for _, file := range data.ImportedFiles {
for _, file := range visibleImports {
cleanFile := stringutil.StripANSI(file)
// Normalize to Unix paths (forward slashes) for cross-platform compatibility
cleanFile = filepath.ToSlash(cleanFile)
Expand Down
Loading
Loading