diff --git a/.gitignore b/.gitignore index c56c11ce63f..3e02f312900 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ node_modules/ gh-aw-test/ pkg/cli/workflows/*.yml + +# Workflow imports (cached files only, not the lock file) +.aw/imports/ diff --git a/cmd/gh-aw/main.go b/cmd/gh-aw/main.go index df3b41a1c87..51bc45cf3c6 100644 --- a/cmd/gh-aw/main.go +++ b/cmd/gh-aw/main.go @@ -229,7 +229,34 @@ Examples: } var installCmd = &cobra.Command{ - Use: "install [@version]", + Use: "install [workflow-name]", + Short: "Install workflow imports and dependencies", + Long: `Install imports for agentic workflows. + +If a workflow name is specified, only imports for that workflow are installed. +Otherwise, all imports from all workflows are installed. + +Examples: + ` + constants.CLIExtensionPrefix + ` install # Install all imports + ` + constants.CLIExtensionPrefix + ` install my-workflow # Install imports for specific workflow`, + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + var workflowName string + if len(args) > 0 { + workflowName = args[0] + } + if err := cli.InstallImports(workflowName, verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{ + Type: "error", + Message: fmt.Sprintf("installing imports: %v", err), + })) + os.Exit(1) + } + }, +} + +var installPackageCmd = &cobra.Command{ + Use: "install-package [@version]", Short: "Install agentic workflows from a GitHub repository", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { @@ -311,8 +338,8 @@ func init() { listCmd.Flags().BoolP("packages", "p", false, "List installed packages instead of available workflows") listCmd.Flags().BoolP("local", "l", false, "List local packages instead of global packages (requires --packages)") - // Add local flag to install command - installCmd.Flags().BoolP("local", "l", false, "Install packages locally in .aw/packages instead of globally in ~/.aw/packages") + // Add local flag to install-package command + installPackageCmd.Flags().BoolP("local", "l", false, "Install packages locally in .aw/packages instead of globally in ~/.aw/packages") // Add local flag to uninstall command uninstallCmd.Flags().BoolP("local", "l", false, "Uninstall packages from local .aw/packages instead of global ~/.aw/packages") @@ -341,6 +368,7 @@ func init() { rootCmd.AddCommand(newCmd) rootCmd.AddCommand(initCmd) rootCmd.AddCommand(installCmd) + rootCmd.AddCommand(installPackageCmd) rootCmd.AddCommand(uninstallCmd) rootCmd.AddCommand(compileCmd) rootCmd.AddCommand(runCmd) diff --git a/docs/src/content/docs/reference/imports.md b/docs/src/content/docs/reference/imports.md new file mode 100644 index 00000000000..75702cf5bed --- /dev/null +++ b/docs/src/content/docs/reference/imports.md @@ -0,0 +1,230 @@ +--- +title: Imports +description: Import workflow components from external GitHub repositories +--- + +The `imports` field allows agentic workflows to import content from external GitHub repositories, enabling code reuse and modular workflow design. + +## Basic Usage + +Import components from external repositories by specifying the repository, version, and file path in the frontmatter: + +```aw +--- +on: workflow_dispatch +permissions: + contents: read +engine: claude +imports: + - microsoft/genaiscript v1.5.0 agentics/engine.md + - githubnext/gh-aw main .github/workflows/agentics/shared/tool-refused.md +--- + +# My Workflow + +This workflow imports shared components from external repositories. +``` + +## Import Format + +Each import follows the format: `org/repo version path` + +- **org/repo**: GitHub repository in `owner/repository` format +- **version**: Git reference (branch name, tag, or commit SHA) +- **path**: Relative path to the file within the repository + +Examples: +- `microsoft/genaiscript v1.5.0 agentics/engine.md` - Import from a specific version tag +- `githubnext/gh-aw main .github/workflows/shared/config.md` - Import from main branch +- `example/repo abc123def path/to/file.md` - Import from specific commit + +## Installation + +Before compiling workflows with imports, install the dependencies: + +```bash +# Install imports for all workflows +gh aw install + +# Install imports for a specific workflow +gh aw install my-workflow +``` + +The install command: +1. Parses import specifications from workflow files +2. Resolves versions to commit SHAs using git ls-remote +3. Clones repositories at the specified versions +4. Stores imported files in `.aw/imports/` +5. Creates/updates `.aw/imports.lock` with resolved SHAs +6. Creates `.aw/.gitignore` to automatically ignore the `imports/` folder + +## Lock File + +The install command generates `.aw/imports.lock` which records: +- Resolved commit SHAs for each import +- Timestamp when imports were resolved +- List of transitive files (from @include directives) + +Example lock file: +``` +# Import lock file generated by gh-aw +# This file records resolved versions and commit SHAs for imports +# Do not edit manually + +version: 1 + +microsoft/genaiscript v1.5.0 agentics/engine.md abc123def456... 2025-01-15T10:30:00Z + +githubnext/gh-aw main .github/workflows/agentics/shared/tool-refused.md ce06aa00a9ce... 2025-01-15T10:30:01Z +``` + +## Compilation + +During compilation, imported files are processed: +1. Frontmatter from imports is merged (tools, engine config, etc.) +2. Markdown content is prepended to the workflow +3. The workflow is compiled as if the content was local + +```bash +# Compile after installing imports +gh aw compile my-workflow +``` + +If imports are not installed, compilation fails with: +``` +error: import org/repo version path not found in lock file (run 'gh aw install') +``` + +## Import Resolution + +Imports are resolved before @include directives during compilation: +1. Parse imports from frontmatter +2. Read lock file and verify imports exist +3. Load imported files from `.aw/imports/` +4. Merge frontmatter and markdown content +5. Process @include directives +6. Compile final workflow + +## Frontmatter Merging + +When importing files with frontmatter, configurations are merged: + +**Tools**: Tool configurations are combined. If the same tool exists in both files, the imported configuration is used. + +```aw +# Imported file +--- +tools: + github: + allowed: [get_issue, list_issues] +--- +``` + +```aw +# Main workflow +--- +imports: + - org/repo v1.0 tools-config.md +tools: + github: + allowed: [get_pull_request] +--- +``` + +Result: GitHub tools will have both sets of allowed functions. + +**Other fields**: Imported values take precedence for non-tools fields. + +## Version Control + +The `gh aw install` command automatically creates a `.aw/.gitignore` file that ignores the `imports/` folder. This ensures the cached imported files are not committed while the lock file remains trackable. + +You can also add `.aw/imports/` to your project's root `.gitignore` for additional protection: + +```gitignore +# Workflow imports (cached files only, not the lock file) +.aw/imports/ +``` + +The lock file (`.aw/imports.lock`) should be committed to version control to ensure reproducible builds by pinning exact commit SHAs, similar to `go.sum` or `package-lock.json`. + +## Best Practices + +1. **Use version tags**: Prefer semantic version tags (v1.0.0) over branch names for stability +2. **Install before compile**: Always run `gh aw install` after adding or updating imports +3. **Commit lock file**: Include `.aw/imports.lock` in version control for reproducibility +4. **Test imports**: Verify imported content compiles successfully +5. **Document dependencies**: Document why each import is needed + +## Transitive Dependencies + +Imported files can contain @include directives. The install command automatically: +- Discovers transitive @include references +- Validates all referenced files exist +- Records them in the lock file + +## Comparison with @include + +| Feature | @include | imports | +|---------|----------|---------| +| Source | Local repository | External repositories | +| Resolution | Compile time | Install time | +| Caching | None (always read) | Cached in .aw/imports/ | +| Version control | Implicit (via git) | Explicit (via lock file) | +| Use case | Project-local reuse | Cross-repo sharing | + +Use @include for local files, imports for external dependencies. + +## Troubleshooting + +**Import not found during compile**: +``` +error: import org/repo version path not found in lock file +``` +Solution: Run `gh aw install` to install imports. + +**Version resolution failed**: +``` +error: failed to resolve version v1.0.0 to commit SHA +``` +Solution: Verify the version/tag/branch exists in the repository. + +**File not found in imported repository**: +``` +error: imported file not found: path/to/file.md +``` +Solution: Check the file path exists in the repository at the specified version. + +## Example: Importing Security Notices + +```aw +--- +on: + issues: + types: [opened] +permissions: + contents: read + issues: write +engine: claude +imports: + - githubnext/gh-aw main .github/workflows/agentics/shared/tool-refused.md + - githubnext/gh-aw main .github/workflows/agentics/shared/xpia.md +tools: + github: + allowed: [get_issue, add_issue_comment] +--- + +# Issue Triage Bot + +This workflow imports security notices and tool usage guidelines. + +Analyze issue #${{ github.event.issue.number }} and provide triage recommendations. +``` + +```bash +# Install and compile +gh aw install +gh aw compile issue-triage +``` + +The compiled workflow will include the imported security notices and XPIA protection guidelines. diff --git a/pkg/cli/install_imports.go b/pkg/cli/install_imports.go new file mode 100644 index 00000000000..4c0b7c3276b --- /dev/null +++ b/pkg/cli/install_imports.go @@ -0,0 +1,453 @@ +package cli + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/githubnext/gh-aw/pkg/console" + "github.com/githubnext/gh-aw/pkg/parser" +) + +// InstallImports installs all imports from workflows +// If workflowName is provided, only install imports for that workflow +func InstallImports(workflowName string, verbose bool) error { + if verbose { + if workflowName != "" { + fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Installing imports for workflow: %s", workflowName))) + } else { + fmt.Fprintln(os.Stderr, console.FormatProgressMessage("Installing imports for all workflows")) + } + } + + // Get workflows directory + workflowsDir := filepath.Join(".github", "workflows") + if _, err := os.Stat(workflowsDir); os.IsNotExist(err) { + return fmt.Errorf("workflows directory not found: %s", workflowsDir) + } + + // Collect all imports from workflows + imports, err := collectImportsFromWorkflows(workflowsDir, workflowName, verbose) + if err != nil { + return fmt.Errorf("failed to collect imports: %w", err) + } + + if len(imports) == 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No imports found in workflows")) + return nil + } + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d unique import(s)", len(imports)))) + } + + // Get or create imports directory + importsDir, err := parser.GetImportsDir() + if err != nil { + return fmt.Errorf("failed to get imports directory: %w", err) + } + + if err := os.MkdirAll(importsDir, 0755); err != nil { + return fmt.Errorf("failed to create imports directory: %w", err) + } + + // Get lock file path + lockFilePath, err := parser.GetImportLockFilePath() + if err != nil { + return fmt.Errorf("failed to get lock file path: %w", err) + } + + // Ensure .aw directory exists + awDir := filepath.Dir(lockFilePath) + if err := os.MkdirAll(awDir, 0755); err != nil { + return fmt.Errorf("failed to create .aw directory: %w", err) + } + + // Create .gitignore in .aw directory to ignore imports folder + if err := ensureAwGitignore(awDir, verbose); err != nil { + return fmt.Errorf("failed to create .aw/.gitignore: %w", err) + } + + // Read existing lock file + lock, err := parser.ReadImportLockFile(lockFilePath) + if err != nil { + return fmt.Errorf("failed to read lock file: %w", err) + } + + // Process each import + for _, importSpec := range imports { + if err := installSingleImport(importSpec, importsDir, lock, verbose); err != nil { + return fmt.Errorf("failed to install import %s: %w", importSpec.String(), err) + } + } + + // Write updated lock file + if err := parser.WriteImportLockFile(lockFilePath, lock); err != nil { + return fmt.Errorf("failed to write lock file: %w", err) + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Successfully installed all imports")) + return nil +} + +// collectImportsFromWorkflows collects all unique imports from workflow files +func collectImportsFromWorkflows(workflowsDir, workflowName string, verbose bool) ([]*parser.ImportSpec, error) { + var allImports []*parser.ImportSpec + seen := make(map[string]bool) + + // Determine which files to process + var filesToProcess []string + if workflowName != "" { + // Process specific workflow + workflowFile := filepath.Join(workflowsDir, workflowName) + if !strings.HasSuffix(workflowFile, ".md") { + workflowFile += ".md" + } + if _, err := os.Stat(workflowFile); os.IsNotExist(err) { + return nil, fmt.Errorf("workflow file not found: %s", workflowFile) + } + filesToProcess = append(filesToProcess, workflowFile) + } else { + // Process all markdown files in workflows directory + err := filepath.Walk(workflowsDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(info.Name(), ".md") { + filesToProcess = append(filesToProcess, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk workflows directory: %w", err) + } + } + + // Extract imports from each file + for _, filePath := range filesToProcess { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Checking %s", filePath))) + } + + content, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", filePath, err) + } + + // Parse frontmatter + result, err := parser.ExtractFrontmatterFromContent(string(content)) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse frontmatter in %s: %v", filePath, err))) + } + continue + } + + // Check for imports field + importsValue, hasImports := result.Frontmatter["imports"] + if !hasImports { + continue + } + + // Parse imports + imports, err := parser.ParseImports(importsValue) + if err != nil { + return nil, fmt.Errorf("failed to parse imports in %s: %w", filePath, err) + } + + // Add to collection (deduplicate) + for _, imp := range imports { + key := imp.String() + if !seen[key] { + seen[key] = true + allImports = append(allImports, imp) + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf(" Found import: %s", imp.String()))) + } + } + } + } + + return allImports, nil +} + +// installSingleImport installs a single import by cloning the repository +func installSingleImport(importSpec *parser.ImportSpec, importsDir string, lock *parser.ImportLockFile, verbose bool) error { + // Check if already installed with same SHA in lock file + existingEntry := lock.FindEntry(importSpec) + + // Get target directory for this import + targetDir := importSpec.GetLocalCachePath(importsDir) + + // Resolve commit SHA + commitSHA, err := resolveImportVersion(importSpec, verbose) + if err != nil { + return fmt.Errorf("failed to resolve version: %w", err) + } + + // Check if already installed with correct SHA + if existingEntry != nil && existingEntry.CommitSHA == commitSHA { + if _, err := os.Stat(targetDir); err == nil { + // Verify the imported file exists + importedFilePath := filepath.Join(targetDir, importSpec.Path) + if _, err := os.Stat(importedFilePath); err == nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Import %s already installed with correct version", importSpec.String()))) + } + return nil + } + } + } + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Installing %s (%s)", importSpec.String(), commitSHA[:8]))) + } + + // Remove existing directory if present + if _, err := os.Stat(targetDir); err == nil { + if err := os.RemoveAll(targetDir); err != nil { + return fmt.Errorf("failed to remove existing import directory: %w", err) + } + } + + // Create parent directory + if err := os.MkdirAll(filepath.Dir(targetDir), 0755); err != nil { + return fmt.Errorf("failed to create parent directory: %w", err) + } + + // Clone repository (shallow clone at specific commit) + repoURL := fmt.Sprintf("https://github.com/%s/%s.git", importSpec.Org, importSpec.Repo) + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Cloning from %s", repoURL))) + } + + // Use git clone with depth 1 at specific commit + cmd := exec.Command("git", "clone", "--depth", "1", "--branch", importSpec.Version, repoURL, targetDir) + output, err := cmd.CombinedOutput() + if err != nil { + // If branch clone fails, try full clone and checkout + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Shallow clone failed, trying full clone...")) + } + + cmd = exec.Command("git", "clone", repoURL, targetDir) + output, err = cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to clone repository: %w (output: %s)", err, string(output)) + } + + // Checkout specific version + cmd = exec.Command("git", "-C", targetDir, "checkout", commitSHA) + output, err = cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to checkout version %s: %w (output: %s)", commitSHA, err, string(output)) + } + } + + // Verify the imported file exists + importedFilePath := filepath.Join(targetDir, importSpec.Path) + if _, err := os.Stat(importedFilePath); os.IsNotExist(err) { + return fmt.Errorf("imported file not found: %s", importSpec.Path) + } + + // Collect transitive files (from @includes in the imported file) + transitiveFiles, err := collectTransitiveFiles(importedFilePath, targetDir, verbose) + if err != nil { + return fmt.Errorf("failed to collect transitive files: %w", err) + } + + // Update lock entry + entry := parser.CreateImportLockEntry(importSpec, commitSHA, transitiveFiles) + lock.AddOrUpdateEntry(entry) + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Installed %s", importSpec.String()))) + } + + return nil +} + +// resolveImportVersion resolves a version string to a commit SHA +func resolveImportVersion(importSpec *parser.ImportSpec, verbose bool) (string, error) { + repoSlug := importSpec.RepoSlug() + version := importSpec.Version + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Resolving version %s for %s", version, repoSlug))) + } + + // Use git ls-remote to resolve the version to a commit SHA + // This works without authentication and is more reliable + repoURL := fmt.Sprintf("https://github.com/%s.git", repoSlug) + + // Try to resolve as a branch or tag + cmd := exec.Command("git", "ls-remote", repoURL, version) + output, err := cmd.CombinedOutput() + if verbose && err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("git ls-remote error: %v, output: %s", err, string(output)))) + } + if err == nil && len(output) > 0 { + // Parse output: SHA \t refs/... + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + parts := strings.Fields(line) + if len(parts) >= 1 { + sha := parts[0] + if len(sha) >= 40 { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Resolved %s to %s", version, sha[:8]))) + } + return sha[:40], nil + } + } + } + } + + // Try as a tag ref + cmd = exec.Command("git", "ls-remote", repoURL, fmt.Sprintf("refs/tags/%s", version)) + output, err = cmd.Output() + if err == nil && len(output) > 0 { + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + parts := strings.Fields(line) + if len(parts) >= 1 { + sha := parts[0] + if len(sha) >= 40 { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Resolved %s to %s", version, sha[:8]))) + } + return sha[:40], nil + } + } + } + } + + // Try as a branch ref + cmd = exec.Command("git", "ls-remote", repoURL, fmt.Sprintf("refs/heads/%s", version)) + output, err = cmd.Output() + if err == nil && len(output) > 0 { + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + parts := strings.Fields(line) + if len(parts) >= 1 { + sha := parts[0] + if len(sha) >= 40 { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Resolved %s to %s", version, sha[:8]))) + } + return sha[:40], nil + } + } + } + } + + // If we still can't resolve, return the version as-is (might be a full SHA) + if len(version) >= 40 && importSpec.IsCommitSHA() { + return version[:40], nil + } + + return "", fmt.Errorf("failed to resolve version %s to commit SHA", version) +} + +// collectTransitiveFiles collects all files referenced by @include directives +func collectTransitiveFiles(filePath, baseDir string, verbose bool) ([]string, error) { + var files []string + seen := make(map[string]bool) + + var collectRecursive func(path string) error + collectRecursive = func(path string) error { + if seen[path] { + return nil + } + seen[path] = true + + // Read file + content, err := os.ReadFile(path) + if err != nil { + return err + } + + // Look for @include directives + lines := strings.Split(string(content), "\n") + + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "@include") { + // Extract include path + parts := strings.Fields(line) + if len(parts) >= 2 { + includePath := parts[1] + + // Handle section references + if strings.Contains(includePath, "#") { + parts := strings.SplitN(includePath, "#", 2) + includePath = parts[0] + } + + // Resolve relative to base directory + fullPath := filepath.Join(baseDir, includePath) + relPath, err := filepath.Rel(baseDir, fullPath) + if err == nil && !strings.HasPrefix(relPath, "..") { + files = append(files, relPath) + // Recursively collect from included file + if err := collectRecursive(fullPath); err != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to process include %s: %v", includePath, err))) + } + } + } + } + } + + return nil + } + + if err := collectRecursive(filePath); err != nil { + return nil, err + } + + return files, nil +} + +// ensureAwGitignore creates or updates .gitignore in .aw directory to ignore imports folder +func ensureAwGitignore(awDir string, verbose bool) error { + gitignorePath := filepath.Join(awDir, ".gitignore") + + // Content to write - ignore the imports folder + content := "# Ignore cached imported files\nimports/\n" + + // Check if .gitignore already exists + existingContent, err := os.ReadFile(gitignorePath) + if err == nil { + // File exists, check if it already has the imports/ entry + if strings.Contains(string(existingContent), "imports/") { + // Already configured, no need to update + return nil + } + // Append to existing content + content = string(existingContent) + "\n" + content + } + + // Write the .gitignore file + if err := os.WriteFile(gitignorePath, []byte(content), 0644); err != nil { + return fmt.Errorf("failed to write .gitignore: %w", err) + } + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Created/updated %s", gitignorePath))) + } + + return nil +} diff --git a/pkg/parser/frontmatter.go b/pkg/parser/frontmatter.go index c0277c6bd39..abf626e4813 100644 --- a/pkg/parser/frontmatter.go +++ b/pkg/parser/frontmatter.go @@ -14,6 +14,139 @@ import ( "github.com/goccy/go-yaml" ) +// ProcessImportsInFrontmatter processes import directives from frontmatter by merging imported content +func ProcessImportsInFrontmatter(content string, frontmatter map[string]any, baseDir string) (string, map[string]any, error) { + // Check if frontmatter has imports + importsValue, hasImports := frontmatter["imports"] + if !hasImports { + return content, frontmatter, nil + } + + // Parse imports + imports, err := ParseImports(importsValue) + if err != nil { + return "", nil, fmt.Errorf("failed to parse imports: %w", err) + } + + if len(imports) == 0 { + return content, frontmatter, nil + } + + // Get imports directory + importsDir, err := GetImportsDir() + if err != nil { + return "", nil, fmt.Errorf("failed to get imports directory: %w", err) + } + + // Get lock file + lockFilePath, err := GetImportLockFilePath() + if err != nil { + return "", nil, fmt.Errorf("failed to get lock file path: %w", err) + } + + lock, err := ReadImportLockFile(lockFilePath) + if err != nil { + return "", nil, fmt.Errorf("failed to read lock file (run 'gh aw install' first): %w", err) + } + + // Process each import + var mergedContent strings.Builder + mergedFrontmatter := make(map[string]any) + + // Copy existing frontmatter (except imports) + for k, v := range frontmatter { + if k != "imports" { + mergedFrontmatter[k] = v + } + } + + for _, importSpec := range imports { + // Find in lock file + entry := lock.FindEntry(importSpec) + if entry == nil { + return "", nil, fmt.Errorf("import %s not found in lock file (run 'gh aw install')", importSpec.String()) + } + + // Get local path + localDir := importSpec.GetLocalCachePath(importsDir) + importedFilePath := filepath.Join(localDir, importSpec.Path) + + // Check if file exists + if _, err := os.Stat(importedFilePath); os.IsNotExist(err) { + return "", nil, fmt.Errorf("imported file not found: %s (run 'gh aw install')", importedFilePath) + } + + // Read imported file + importedContent, err := os.ReadFile(importedFilePath) + if err != nil { + return "", nil, fmt.Errorf("failed to read imported file %s: %w", importedFilePath, err) + } + + // Parse imported file + importedResult, err := ExtractFrontmatterFromContent(string(importedContent)) + if err != nil { + return "", nil, fmt.Errorf("failed to parse imported file %s: %w", importedFilePath, err) + } + + // Merge frontmatter (tools, etc.) + if err := mergeFrontmatterMaps(mergedFrontmatter, importedResult.Frontmatter); err != nil { + return "", nil, fmt.Errorf("failed to merge frontmatter from import %s: %w", importSpec.String(), err) + } + + // Append markdown content + if importedResult.Markdown != "" { + mergedContent.WriteString(importedResult.Markdown) + mergedContent.WriteString("\n\n") + } + } + + // Append original content + mergedContent.WriteString(content) + + return mergedContent.String(), mergedFrontmatter, nil +} + +// mergeFrontmatterMaps merges two frontmatter maps +func mergeFrontmatterMaps(target, source map[string]any) error { + for key, sourceValue := range source { + // Skip imports field + if key == "imports" { + continue + } + + targetValue, exists := target[key] + if !exists { + // Key doesn't exist in target, add it + target[key] = sourceValue + continue + } + + // Key exists, need to merge + switch key { + case "tools": + // Merge tools + targetToolsMap, ok := targetValue.(map[string]any) + if !ok { + return fmt.Errorf("target tools is not a map") + } + + sourceToolsMap, ok := sourceValue.(map[string]any) + if !ok { + return fmt.Errorf("source tools is not a map") + } + + // Merge tool configurations + for toolName, sourceToolConfig := range sourceToolsMap { + targetToolsMap[toolName] = sourceToolConfig + } + default: + // For other keys, imported value takes precedence + target[key] = sourceValue + } + } + return nil +} + // isMCPType checks if a type string represents an MCP-compatible type func isMCPType(typeStr string) bool { switch typeStr { diff --git a/pkg/parser/imports.go b/pkg/parser/imports.go new file mode 100644 index 00000000000..62591e79bd1 --- /dev/null +++ b/pkg/parser/imports.go @@ -0,0 +1,164 @@ +package parser + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" +) + +// ImportSpec represents a single import specification from frontmatter +// Format: "org/repo version path" (go.mod style) +type ImportSpec struct { + Org string // GitHub organization + Repo string // Repository name + Version string // Version/tag/branch/commit + Path string // Path to the file within the repository +} + +// ParseImportSpec parses a single import string into an ImportSpec +// Expected format: "org/repo version path" +// Example: "microsoft/genaiscript v1.5 agentics/engine.md" +func ParseImportSpec(importStr string) (*ImportSpec, error) { + // Trim whitespace + importStr = strings.TrimSpace(importStr) + if importStr == "" { + return nil, fmt.Errorf("empty import specification") + } + + // Split by whitespace + parts := strings.Fields(importStr) + if len(parts) < 3 { + return nil, fmt.Errorf("invalid import format '%s': expected 'org/repo version path'", importStr) + } + + // Parse org/repo + repoSpec := parts[0] + if !strings.Contains(repoSpec, "/") { + return nil, fmt.Errorf("invalid repository format '%s': expected 'org/repo'", repoSpec) + } + repoParts := strings.SplitN(repoSpec, "/", 2) + if len(repoParts) != 2 { + return nil, fmt.Errorf("invalid repository format '%s': expected 'org/repo'", repoSpec) + } + + version := parts[1] + path := strings.Join(parts[2:], " ") // Allow spaces in path + + // Validate version format (basic check) + if version == "" { + return nil, fmt.Errorf("empty version in import specification") + } + + // Validate path + if path == "" { + return nil, fmt.Errorf("empty path in import specification") + } + + return &ImportSpec{ + Org: repoParts[0], + Repo: repoParts[1], + Version: version, + Path: path, + }, nil +} + +// ParseImports parses an array of import strings from frontmatter +func ParseImports(importsValue interface{}) ([]*ImportSpec, error) { + if importsValue == nil { + return nil, nil + } + + // Handle array of strings + importsArray, ok := importsValue.([]interface{}) + if !ok { + return nil, fmt.Errorf("imports must be an array of strings") + } + + var imports []*ImportSpec + for i, item := range importsArray { + importStr, ok := item.(string) + if !ok { + return nil, fmt.Errorf("import at index %d is not a string", i) + } + + spec, err := ParseImportSpec(importStr) + if err != nil { + return nil, fmt.Errorf("import at index %d: %w", i, err) + } + + imports = append(imports, spec) + } + + return imports, nil +} + +// String returns the import specification as a string +func (i *ImportSpec) String() string { + return fmt.Sprintf("%s/%s %s %s", i.Org, i.Repo, i.Version, i.Path) +} + +// RepoSlug returns the repository slug (org/repo) +func (i *ImportSpec) RepoSlug() string { + return fmt.Sprintf("%s/%s", i.Org, i.Repo) +} + +// ImportedFilePath returns the expected file path within the imported repository +func (i *ImportSpec) ImportedFilePath() string { + return i.Path +} + +// ImportLockEntry represents an entry in the import lock file +type ImportLockEntry struct { + ImportSpec *ImportSpec + CommitSHA string // Resolved commit SHA + ResolvedAt string // Timestamp when resolved + Files []string // List of transitive files included +} + +// ImportLockFile represents the complete lock file structure +type ImportLockFile struct { + Version string // Lock file format version + Entries []*ImportLockEntry // Locked import entries +} + +// ValidatePath checks if a path looks valid +func (i *ImportSpec) ValidatePath() error { + // Basic validation + if strings.HasPrefix(i.Path, "/") { + return fmt.Errorf("path must be relative, not absolute: %s", i.Path) + } + if strings.Contains(i.Path, "..") { + return fmt.Errorf("path must not contain '..': %s", i.Path) + } + return nil +} + +// IsVersionTag checks if the version looks like a semver tag +func (i *ImportSpec) IsVersionTag() bool { + // Check if version starts with 'v' followed by digits + matched, _ := regexp.MatchString(`^v\d+`, i.Version) + return matched +} + +// IsCommitSHA checks if the version looks like a commit SHA +func (i *ImportSpec) IsCommitSHA() bool { + // Check if version is a 40-character hex string (full SHA) + matched, _ := regexp.MatchString(`^[0-9a-f]{40}$`, i.Version) + return matched +} + +// NormalizeVersion normalizes the version string for consistent comparison +func (i *ImportSpec) NormalizeVersion() string { + // Remove 'refs/tags/' or 'refs/heads/' prefix if present + version := i.Version + version = strings.TrimPrefix(version, "refs/tags/") + version = strings.TrimPrefix(version, "refs/heads/") + return version +} + +// GetLocalCachePath returns the local cache path for this import +func (i *ImportSpec) GetLocalCachePath(importsDir string) string { + // Store in .aw/imports/org/repo/version/ + return filepath.Join(importsDir, i.Org, i.Repo, i.Version) +} diff --git a/pkg/parser/imports_lock.go b/pkg/parser/imports_lock.go new file mode 100644 index 00000000000..6a028a2d8c2 --- /dev/null +++ b/pkg/parser/imports_lock.go @@ -0,0 +1,254 @@ +package parser + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// ImportLockFileVersion is the current version of the lock file format +const ImportLockFileVersion = "1" + +// ReadImportLockFile reads and parses an import lock file +func ReadImportLockFile(lockFilePath string) (*ImportLockFile, error) { + if _, err := os.Stat(lockFilePath); os.IsNotExist(err) { + // Lock file doesn't exist, return empty lock + return &ImportLockFile{ + Version: ImportLockFileVersion, + Entries: []*ImportLockEntry{}, + }, nil + } + + file, err := os.Open(lockFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open lock file: %w", err) + } + defer file.Close() + + lock := &ImportLockFile{ + Version: ImportLockFileVersion, + Entries: []*ImportLockEntry{}, + } + + scanner := bufio.NewScanner(file) + var currentEntry *ImportLockEntry + var inFilesSection bool + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + // Skip empty lines and comments + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + // Check for version line + if strings.HasPrefix(line, "version:") { + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + lock.Version = strings.TrimSpace(parts[1]) + } + continue + } + + // Check for import line (starts with org/repo) + if strings.Contains(line, " ") && !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") { + // Save previous entry if exists + if currentEntry != nil { + lock.Entries = append(lock.Entries, currentEntry) + } + + // Parse new entry + parts := strings.Fields(line) + if len(parts) >= 4 { + spec, err := ParseImportSpec(strings.Join(parts[:3], " ")) + if err != nil { + return nil, fmt.Errorf("failed to parse import spec '%s': %w", line, err) + } + + currentEntry = &ImportLockEntry{ + ImportSpec: spec, + CommitSHA: parts[3], + ResolvedAt: "", + Files: []string{}, + } + + // Check for resolved timestamp (optional, after SHA) + if len(parts) > 4 { + currentEntry.ResolvedAt = strings.Join(parts[4:], " ") + } + + inFilesSection = false + } + continue + } + + // Check for files section + if strings.HasPrefix(line, " files:") || strings.HasPrefix(line, "\tfiles:") { + inFilesSection = true + continue + } + + // Parse file entry + if inFilesSection && currentEntry != nil { + // Remove leading whitespace/dash + file := strings.TrimSpace(line) + file = strings.TrimPrefix(file, "- ") + file = strings.TrimPrefix(file, "-") + file = strings.TrimSpace(file) + if file != "" { + currentEntry.Files = append(currentEntry.Files, file) + } + } + } + + // Add last entry + if currentEntry != nil { + lock.Entries = append(lock.Entries, currentEntry) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed to read lock file: %w", err) + } + + return lock, nil +} + +// WriteImportLockFile writes an import lock file +func WriteImportLockFile(lockFilePath string, lock *ImportLockFile) error { + // Sort entries by repo slug for consistent output + sort.Slice(lock.Entries, func(i, j int) bool { + return lock.Entries[i].ImportSpec.RepoSlug() < lock.Entries[j].ImportSpec.RepoSlug() + }) + + file, err := os.Create(lockFilePath) + if err != nil { + return fmt.Errorf("failed to create lock file: %w", err) + } + defer file.Close() + + // Write header + fmt.Fprintf(file, "# Import lock file generated by gh-aw\n") + fmt.Fprintf(file, "# This file records resolved versions and commit SHAs for imports\n") + fmt.Fprintf(file, "# Do not edit manually\n\n") + fmt.Fprintf(file, "version: %s\n\n", lock.Version) + + // Write entries + for _, entry := range lock.Entries { + // Write import spec with commit SHA + fmt.Fprintf(file, "%s %s", entry.ImportSpec.String(), entry.CommitSHA) + + // Write resolved timestamp if available + if entry.ResolvedAt != "" { + fmt.Fprintf(file, " %s", entry.ResolvedAt) + } + fmt.Fprintf(file, "\n") + + // Write files section if present + if len(entry.Files) > 0 { + fmt.Fprintf(file, " files:\n") + for _, f := range entry.Files { + fmt.Fprintf(file, " - %s\n", f) + } + } + + fmt.Fprintf(file, "\n") + } + + return nil +} + +// FindEntry finds a lock entry by import spec +func (l *ImportLockFile) FindEntry(spec *ImportSpec) *ImportLockEntry { + for _, entry := range l.Entries { + if entry.ImportSpec.RepoSlug() == spec.RepoSlug() && + entry.ImportSpec.Version == spec.Version && + entry.ImportSpec.Path == spec.Path { + return entry + } + } + return nil +} + +// AddOrUpdateEntry adds or updates a lock entry +func (l *ImportLockFile) AddOrUpdateEntry(entry *ImportLockEntry) { + // Check if entry exists + for i, existing := range l.Entries { + if existing.ImportSpec.RepoSlug() == entry.ImportSpec.RepoSlug() && + existing.ImportSpec.Version == entry.ImportSpec.Version && + existing.ImportSpec.Path == entry.ImportSpec.Path { + // Update existing entry + l.Entries[i] = entry + return + } + } + + // Add new entry + l.Entries = append(l.Entries, entry) +} + +// GetImportsDir returns the directory where imports are cached +func GetImportsDir() (string, error) { + // Use .aw/imports in the current project + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current directory: %w", err) + } + + // Find git root (go up until we find .git directory) + gitRoot, err := findGitRoot(cwd) + if err != nil { + return "", fmt.Errorf("not in a git repository: %w", err) + } + + importsDir := filepath.Join(gitRoot, ".aw", "imports") + return importsDir, nil +} + +// GetImportLockFilePath returns the path to the import lock file +func GetImportLockFilePath() (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current directory: %w", err) + } + + gitRoot, err := findGitRoot(cwd) + if err != nil { + return "", fmt.Errorf("not in a git repository: %w", err) + } + + lockFile := filepath.Join(gitRoot, ".aw", "imports.lock") + return lockFile, nil +} + +// findGitRoot finds the git root directory by looking for .git +func findGitRoot(startDir string) (string, error) { + dir := startDir + for { + gitDir := filepath.Join(dir, ".git") + if _, err := os.Stat(gitDir); err == nil { + return dir, nil + } + + parent := filepath.Dir(dir) + if parent == dir { + // Reached root without finding .git + return "", fmt.Errorf(".git directory not found") + } + dir = parent + } +} + +// CreateImportLockEntry creates a new lock entry with current timestamp +func CreateImportLockEntry(spec *ImportSpec, commitSHA string, files []string) *ImportLockEntry { + return &ImportLockEntry{ + ImportSpec: spec, + CommitSHA: commitSHA, + ResolvedAt: time.Now().UTC().Format(time.RFC3339), + Files: files, + } +} diff --git a/pkg/parser/imports_test.go b/pkg/parser/imports_test.go new file mode 100644 index 00000000000..bf87c89269f --- /dev/null +++ b/pkg/parser/imports_test.go @@ -0,0 +1,316 @@ +package parser + +import ( + "testing" +) + +func TestParseImportSpec(t *testing.T) { + tests := []struct { + name string + input string + wantOrg string + wantRepo string + wantVersion string + wantPath string + wantErr bool + }{ + { + name: "valid import with single path segment", + input: "microsoft/genaiscript v1.5 agentics/engine.md", + wantOrg: "microsoft", + wantRepo: "genaiscript", + wantVersion: "v1.5", + wantPath: "agentics/engine.md", + wantErr: false, + }, + { + name: "valid import with nested path", + input: "github/copilot v2.0.0 workflows/shared/config.md", + wantOrg: "github", + wantRepo: "copilot", + wantVersion: "v2.0.0", + wantPath: "workflows/shared/config.md", + wantErr: false, + }, + { + name: "valid import with branch name", + input: "githubnext/gh-aw main README.md", + wantOrg: "githubnext", + wantRepo: "gh-aw", + wantVersion: "main", + wantPath: "README.md", + wantErr: false, + }, + { + name: "valid import with commit SHA", + input: "example/repo abc123def456 path/to/file.md", + wantOrg: "example", + wantRepo: "repo", + wantVersion: "abc123def456", + wantPath: "path/to/file.md", + wantErr: false, + }, + { + name: "invalid: missing path", + input: "microsoft/genaiscript v1.5", + wantErr: true, + }, + { + name: "invalid: missing version and path", + input: "microsoft/genaiscript", + wantErr: true, + }, + { + name: "invalid: no org slash", + input: "microsoft-genaiscript v1.5 path.md", + wantErr: true, + }, + { + name: "invalid: empty string", + input: "", + wantErr: true, + }, + { + name: "invalid: only whitespace", + input: " ", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec, err := ParseImportSpec(tt.input) + + if tt.wantErr { + if err == nil { + t.Errorf("ParseImportSpec() expected error, got nil") + } + return + } + + if err != nil { + t.Fatalf("ParseImportSpec() unexpected error: %v", err) + } + + if spec.Org != tt.wantOrg { + t.Errorf("ParseImportSpec() org = %v, want %v", spec.Org, tt.wantOrg) + } + if spec.Repo != tt.wantRepo { + t.Errorf("ParseImportSpec() repo = %v, want %v", spec.Repo, tt.wantRepo) + } + if spec.Version != tt.wantVersion { + t.Errorf("ParseImportSpec() version = %v, want %v", spec.Version, tt.wantVersion) + } + if spec.Path != tt.wantPath { + t.Errorf("ParseImportSpec() path = %v, want %v", spec.Path, tt.wantPath) + } + }) + } +} + +func TestImportSpec_RepoSlug(t *testing.T) { + spec := &ImportSpec{ + Org: "microsoft", + Repo: "genaiscript", + Version: "v1.5", + Path: "agentics/engine.md", + } + + expected := "microsoft/genaiscript" + if spec.RepoSlug() != expected { + t.Errorf("RepoSlug() = %v, want %v", spec.RepoSlug(), expected) + } +} + +func TestImportSpec_String(t *testing.T) { + spec := &ImportSpec{ + Org: "microsoft", + Repo: "genaiscript", + Version: "v1.5", + Path: "agentics/engine.md", + } + + expected := "microsoft/genaiscript v1.5 agentics/engine.md" + if spec.String() != expected { + t.Errorf("String() = %v, want %v", spec.String(), expected) + } +} + +func TestImportSpec_ValidatePath(t *testing.T) { + tests := []struct { + name string + path string + wantErr bool + }{ + { + name: "valid relative path", + path: "agentics/engine.md", + wantErr: false, + }, + { + name: "invalid absolute path", + path: "/agentics/engine.md", + wantErr: true, + }, + { + name: "invalid path with ..", + path: "../agentics/engine.md", + wantErr: true, + }, + { + name: "valid path with subdirs", + path: "workflows/shared/config.md", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := &ImportSpec{ + Org: "test", + Repo: "repo", + Version: "v1.0", + Path: tt.path, + } + + err := spec.ValidatePath() + if (err != nil) != tt.wantErr { + t.Errorf("ValidatePath() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestImportSpec_IsVersionTag(t *testing.T) { + tests := []struct { + name string + version string + want bool + }{ + { + name: "semver tag v1.5", + version: "v1.5", + want: true, + }, + { + name: "semver tag v2.0.0", + version: "v2.0.0", + want: true, + }, + { + name: "branch name main", + version: "main", + want: false, + }, + { + name: "commit SHA", + version: "abc123def456", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := &ImportSpec{ + Org: "test", + Repo: "repo", + Version: tt.version, + Path: "path.md", + } + + if got := spec.IsVersionTag(); got != tt.want { + t.Errorf("IsVersionTag() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseImports(t *testing.T) { + tests := []struct { + name string + input interface{} + wantCount int + wantErr bool + }{ + { + name: "valid array of imports", + input: []interface{}{ + "microsoft/genaiscript v1.5 agentics/engine.md", + "github/copilot v2.0.0 workflows/shared/config.md", + }, + wantCount: 2, + wantErr: false, + }, + { + name: "nil input", + input: nil, + wantCount: 0, + wantErr: false, + }, + { + name: "empty array", + input: []interface{}{}, + wantCount: 0, + wantErr: false, + }, + { + name: "invalid: not an array", + input: "microsoft/genaiscript v1.5 agentics/engine.md", + wantErr: true, + }, + { + name: "invalid: array with non-string", + input: []interface{}{ + "microsoft/genaiscript v1.5 agentics/engine.md", + 123, + }, + wantErr: true, + }, + { + name: "invalid: array with invalid import spec", + input: []interface{}{ + "microsoft/genaiscript v1.5 agentics/engine.md", + "invalid-spec", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + imports, err := ParseImports(tt.input) + + if tt.wantErr { + if err == nil { + t.Errorf("ParseImports() expected error, got nil") + } + return + } + + if err != nil { + t.Fatalf("ParseImports() unexpected error: %v", err) + } + + if len(imports) != tt.wantCount { + t.Errorf("ParseImports() count = %v, want %v", len(imports), tt.wantCount) + } + }) + } +} + +func TestImportSpec_GetLocalCachePath(t *testing.T) { + spec := &ImportSpec{ + Org: "microsoft", + Repo: "genaiscript", + Version: "v1.5", + Path: "agentics/engine.md", + } + + importsDir := "/home/user/.aw/imports" + expected := "/home/user/.aw/imports/microsoft/genaiscript/v1.5" + + got := spec.GetLocalCachePath(importsDir) + if got != expected { + t.Errorf("GetLocalCachePath() = %v, want %v", got, expected) + } +} diff --git a/pkg/parser/schemas/main_workflow_schema.json b/pkg/parser/schemas/main_workflow_schema.json index 010be53041d..e77d232977f 100644 --- a/pkg/parser/schemas/main_workflow_schema.json +++ b/pkg/parser/schemas/main_workflow_schema.json @@ -1334,6 +1334,14 @@ } ] }, + "imports": { + "type": "array", + "description": "Import workflow components from external GitHub repositories. Format: 'org/repo version path' (e.g., 'microsoft/genaiscript v1.5 agentics/engine.md'). Run 'gh aw install' to download imported files before compilation.", + "items": { + "type": "string", + "description": "Import specification in format: 'org/repo version path'" + } + }, "safe-outputs": { "type": "object", "description": "Safe output processing configuration that automatically creates GitHub issues, comments, and pull requests from AI workflow output without requiring write permissions in the main job", diff --git a/pkg/workflow/compiler.go b/pkg/workflow/compiler.go index 91019579395..31a46d7e22c 100644 --- a/pkg/workflow/compiler.go +++ b/pkg/workflow/compiler.go @@ -430,6 +430,30 @@ func (c *Compiler) ParseWorkflowFile(markdownPath string) (*WorkflowData, error) markdownDir := filepath.Dir(markdownPath) + // Process imports from frontmatter if present + if _, hasImports := result.Frontmatter["imports"]; hasImports { + if c.verbose { + fmt.Println(console.FormatInfoMessage("Processing imports...")) + } + + processedMarkdown, processedFrontmatter, err := parser.ProcessImportsInFrontmatter( + result.Markdown, + result.Frontmatter, + markdownDir, + ) + if err != nil { + return nil, fmt.Errorf("failed to process imports: %w", err) + } + + // Update result with processed content + result.Markdown = processedMarkdown + result.Frontmatter = processedFrontmatter + + if c.verbose { + fmt.Println(console.FormatSuccessMessage("Successfully processed imports")) + } + } + // Extract AI engine setting from frontmatter engineSetting, engineConfig := c.extractEngineConfig(result.Frontmatter)