diff --git a/.github/prompts/generate-improvements.prompt.md b/.github/prompts/generate-improvements.prompt.md index 496f2c3..fdadba7 100644 --- a/.github/prompts/generate-improvements.prompt.md +++ b/.github/prompts/generate-improvements.prompt.md @@ -2,7 +2,7 @@ description: Suggest improvements to the AgentRC CLI project across features, bug fixes, security, performance, and engineering quality. --- -You are a senior software engineer reviewing the **AgentRC** project — a TypeScript CLI tool that primes repositories for AI-assisted development by analyzing codebases, generating Copilot instructions and VS Code configs, running evaluations, and producing AI readiness reports. +You are a senior software engineer reviewing the **AgentRC** project — a TypeScript CLI tool that primes repositories for AI-assisted development by analyzing codebases, generating instructions and VS Code configs, running evaluations, and producing readiness reports. ## Architecture Context @@ -18,7 +18,7 @@ You are a senior software engineer reviewing the **AgentRC** project — a TypeS - `instructions.ts` — Generates `.github/copilot-instructions.md` using Copilot SDK agent sessions - `generator.ts` — Writes `.vscode/settings.json` and `.vscode/mcp.json` configs - `evaluator.ts` — Runs eval cases comparing agent responses with/without instructions, builds trajectory viewer HTML - - `readiness.ts` — Multi-pillar AI readiness assessment (style, build, testing, docs, dev-env, code-quality, observability, security, ai-tooling) + - `readiness.ts` — Multi-pillar readiness assessment (style, build, testing, docs, dev-env, code-quality, observability, security, ai-tooling) - `visualReport.ts` — Generates beautiful HTML readiness reports with summary cards, pillar charts, level distribution - `git.ts` — Clone/branch operations via `simple-git` - `github.ts` / `azureDevops.ts` — GitHub (Octokit) and Azure DevOps API integrations @@ -35,7 +35,7 @@ You are a senior software engineer reviewing the **AgentRC** project — a TypeS | `agentrc generate ` | Generate `instructions`, `agents`, `mcp`, or `vscode` configs | | `agentrc instructions` | Generate copilot-instructions.md via Copilot SDK | | `agentrc eval` | Run evaluation cases comparing with/without instructions | -| `agentrc readiness` | AI readiness assessment with optional visual HTML report | +| `agentrc readiness` | Readiness assessment with optional visual HTML report | | `agentrc batch` | Batch process multiple repos across GitHub/Azure orgs | | `agentrc batch-readiness` | Batch readiness reports across multiple repos | | `agentrc pr` | Automate branch/PR creation for generated configs | diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index e7029d1..09a2177 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -100,11 +100,21 @@ jobs: const fs = require('fs'); let summary = ''; + let hasResults = false; let isPR = context.eventName === 'pull_request'; + // Skip reporting when the eval step crashed (no results produced) + if (!fs.existsSync('.agentrc/evals/results.json')) { + const evalOutcome = '${{ steps.eval.outcome }}'; + core.warning(`Eval step finished with outcome "${evalOutcome}" — no results file produced.`); + // Don't post an unhelpful comment to the PR + return; + } + try { const raw = fs.readFileSync('.agentrc/evals/results.json', 'utf8'); const data = JSON.parse(raw); + hasResults = true; const results = data.results || []; const total = results.length; const passed = results.filter(r => r.verdict === 'pass').length; @@ -149,14 +159,14 @@ jobs: summary += `\n\n\n`; } } catch (err) { - summary += `## ⚠️ AgentRC Eval\n\nCould not read eval results: ${err.message}\n`; + summary += `## ⚠️ AgentRC Eval\n\nFailed to parse eval results: ${err.message}\n`; } // Write to Actions job summary (visible in run UI and PR checks tab) await core.summary.addRaw(summary).write(); - // Also post/update as PR comment - if (isPR) { + // Only post/update PR comment when we have actual results + if (isPR && hasResults) { const marker = ''; const body = marker + '\n' + summary; diff --git a/README.md b/README.md index 60757eb..2e37a6d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ AI coding agents are only as effective as the context they receive. AgentRC is a CLI and VS Code extension that closes the gap — from a single repo to hundreds across your org. -**Measure** — Analyze repo structure and score AI readiness across a 5-level maturity model. +**Measure** — Analyze repo structure and score readiness across a 5-level maturity model. **Generate** — Produce tailored instructions, evals, and dev configs using the Copilot SDK. **Maintain** — Run evaluations in CI to catch instruction drift as code evolves. @@ -37,7 +37,7 @@ agentrc analyze # 2. Check how AI-ready your repo is agentrc readiness -# 3. Generate AI instructions +# 3. Generate instructions agentrc instructions # 4. Generate MCP and VS Code configs @@ -72,7 +72,7 @@ agentrc analyze --output analysis.md # save Markdown report agentrc analyze --output analysis.json --force # overwrite existing report ``` -### `agentrc readiness` — Assess AI Readiness +### `agentrc readiness` — Run Readiness Report Score a repo across 9 pillars grouped into **Repo Health** and **AI Setup**: @@ -90,30 +90,37 @@ agentrc readiness --fail-level 3 # CI gate: exit 1 if below level 3 **Maturity levels:** -| Level | Name | What it means | -| ----- | ---------- | -------------------------------------------------- | -| 1 | Functional | Builds, tests, basic tooling in place | -| 2 | Documented | README, CONTRIBUTING, custom AI instructions exist | +| Level | Name | What it means | +| ----- | ------------ | --------------------------------------------------- | +| 1 | Functional | Builds, tests, basic tooling in place | +| 2 | Documented | README, CONTRIBUTING, custom instructions exist | +| 3 | Standardized | CI/CD, security policies, CODEOWNERS, observability | +| 4 | Optimized | MCP servers, custom agents, AI skills configured | +| 5 | Autonomous | Full AI-native development with minimal oversight | At Level 2, AgentRC also checks **instruction consistency** — when a repo has multiple AI instruction files (e.g. `copilot-instructions.md`, `CLAUDE.md`, `AGENTS.md`), it detects whether they diverge. Symlinked or identical files pass; diverging files fail with a similarity score and a suggestion to consolidate. -| 3 | Standardized | CI/CD, security policies, CODEOWNERS, observability | -| 4 | Optimized | MCP servers, custom agents, AI skills configured | -| 5 | Autonomous | Full AI-native development with minimal oversight | - ### `agentrc instructions` — Generate Instructions -Generate `copilot-instructions.md` or `AGENTS.md` using the Copilot SDK: +Generate instructions using the Copilot SDK: ```bash agentrc instructions # copilot-instructions.md (default) -agentrc instructions --format agents-md # AGENTS.md -agentrc instructions --per-app # per-app in monorepos +agentrc instructions --output AGENTS.md # custom output path +agentrc instructions --strategy nested # nested hub + detail files in .agents/ agentrc instructions --areas # root + all detected areas agentrc instructions --area frontend # single area -agentrc instructions --model claude-sonnet-4.5 +agentrc instructions --areas-only # areas only (skip root) +agentrc instructions --dry-run # preview without writing +agentrc instructions --model claude-sonnet-4.6 ``` +**Concepts:** + +- **Format**: Output file — `copilot-instructions.md` (default) or `AGENTS.md` (via `--output`) +- **Strategy**: `flat` (single file, default) or `nested` (hub + per-topic detail files) +- **Scope**: `root only` (default), `--areas` (root + areas), `--area ` (single area) + ### `agentrc eval` — Evaluate Instructions Measure how instructions improve AI responses with a judge model: @@ -121,12 +128,14 @@ Measure how instructions improve AI responses with a judge model: ```bash agentrc eval --init # scaffold eval config from codebase agentrc eval agentrc.eval.json # run evaluation -agentrc eval --model gpt-4.1 --judge-model claude-sonnet-4.5 +agentrc eval --model gpt-4.1 --judge-model claude-sonnet-4.6 agentrc eval --fail-level 80 # CI gate: exit 1 if pass rate < 80% ``` ### `agentrc generate` — Generate Configs +> **Note:** `generate instructions` and `generate agents` are deprecated — use `agentrc instructions` directly. + ```bash agentrc generate mcp # .vscode/mcp.json agentrc generate vscode --force # .vscode/settings.json (overwrite) @@ -148,9 +157,9 @@ agentrc pr owner/repo-name # clone → generate → open PR agentrc tui ``` -### `agentrc init` — Guided Setup +### `agentrc init` — Init Repository -Interactive or headless repo onboarding — detects your stack and walks through readiness, instructions, and config generation. For monorepos, auto-detects workspaces and bootstraps `agentrc.config.json` with workspace and area definitions. +Interactive or headless repo onboarding — analyzes your stack and generates instructions. For monorepos, auto-detects workspaces and bootstraps `agentrc.config.json` with workspace and area definitions. ### Global Options diff --git a/agentrc.eval.json b/agentrc.eval.json index 118f2ca..06736da 100644 --- a/agentrc.eval.json +++ b/agentrc.eval.json @@ -19,12 +19,12 @@ }, { "id": "case-4", - "prompt": "How does the AI readiness assessment work, and how can it be customized with policies?", - "expectation": "The readiness service in src/services/readiness.ts evaluates repositories across 9 pillars (style-validation, build-system, testing, documentation, dev-environment, code-quality, observability, security, ai-tooling) and assigns a maturity level from 1 (Functional) to 5 (Autonomous). Each criterion has a scope — repo, app, or area — determining whether it runs once, per monorepo app, or per detected area. buildCriteria() returns 20+ built-in checks and buildExtras() adds optional ones. Policies loaded via src/services/policy.ts can customize the assessment: loadPolicy() reads JSON/TS/JS configs, and resolveChain() merges a chain of policies that can disable, override, or add criteria and set pass-rate thresholds. Results can be rendered as an interactive HTML report by src/services/visualReport.ts with dark/light theme toggle and expandable per-pillar details." + "prompt": "How does the readiness assessment work, and how can it be customized with policies?", + "expectation": "The readiness service evaluates repositories across 9 pillars (style-validation, build-system, testing, documentation, dev-environment, code-quality, observability, security, ai-tooling) and assigns a maturity level from 1 (Functional) to 5 (Autonomous). Each criterion has a scope — repo, app, or area — determining whether it runs once, per monorepo app, or per detected area. buildCriteria() returns 20+ built-in checks and buildExtras() adds optional ones. Policies loaded via src/services/policy.ts can customize the assessment: loadPolicy() reads JSON/TS/JS configs, and resolveChain() merges a chain of policies that can disable, override, or add criteria and set pass-rate thresholds. Results can be rendered as an interactive HTML report by src/services/visualReport.ts with dark/light theme toggle and expandable per-pillar details." }, { "id": "case-5", - "prompt": "How does AgentRC generate Copilot instructions, including for monorepos with multiple areas?", + "prompt": "How does AgentRC generate instructions, including for monorepos with multiple areas?", "expectation": "The instruction generation pipeline starts with the analyzer (src/services/analyzer.ts) which scans the repo to detect languages, frameworks, monorepo apps, and logical areas (frontend, backend, etc.) with glob patterns. For root-level instructions, generateCopilotInstructions() in src/services/instructions.ts creates a Copilot SDK session that explores the codebase using tools (glob, view, grep) and produces .github/copilot-instructions.md. For area-specific instructions, generateAreaInstructions() generates focused content per area, and buildAreaFrontmatter() creates YAML frontmatter with applyTo glob patterns so VS Code scopes them to the right files. These are written to .github/instructions/{sanitized-name}.instructions.md via writeAreaInstruction(). The instructions command supports --areas to generate all area instructions, --areas-only to skip the root file, and --area for a single area." }, { diff --git a/docs/product.md b/docs/product.md index ef8ece7..7ace8f4 100644 --- a/docs/product.md +++ b/docs/product.md @@ -10,7 +10,7 @@ This gap widens at scale. An organization with hundreds of repos can't manually - **Platform engineering teams** rolling out AI coding tools across an org — need to assess readiness, set baselines, and track adoption at scale. - **Individual developers** who want their AI agent to understand their repo's stack, conventions, and architecture from day one. -- **Engineering leadership** evaluating AI readiness across portfolios, with quantifiable maturity levels and policy-driven compliance. +- **Engineering leadership** evaluating readiness across portfolios, with quantifiable maturity levels and policy-driven compliance. ## What AgentRC Does @@ -33,7 +33,7 @@ AgentRC's readiness assessment maps repos to a 5-level maturity model: | Level | Name | What it means | | ----- | ------------ | --------------------------------------------------- | | 1 | Functional | Builds, tests, basic tooling in place | -| 2 | Documented | README, CONTRIBUTING, custom AI instructions exist | +| 2 | Documented | README, CONTRIBUTING, custom instructions exist | | 3 | Standardized | CI/CD, security policies, CODEOWNERS, observability | | 4 | Optimized | MCP servers, custom agents, AI skills configured | | 5 | Autonomous | Full AI-native development with minimal oversight | diff --git a/examples/README.md b/examples/README.md index 51b884c..c429180 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,7 +17,7 @@ agentrc init /path/to/repo # Inspect a repository before generating config or instructions agentrc analyze /path/to/repo -# Check AI readiness in the terminal +# Check readiness in the terminal agentrc readiness /path/to/repo # Generate a visual readiness report diff --git a/packages/core/src/services/batch.ts b/packages/core/src/services/batch.ts index ca1b6ff..80f92b5 100644 --- a/packages/core/src/services/batch.ts +++ b/packages/core/src/services/batch.ts @@ -129,7 +129,7 @@ async function processRepo(params: ProcessRepoParams): Promise { } progress?.update(`${label}: Committing...`); - await commitAll(repoPath, "chore: add copilot instructions via AgentRC"); + await commitAll(repoPath, "chore: add instructions via AgentRC"); progress?.update(`${label}: Pushing...`); await pushBranch(repoPath, branch, token, provider); @@ -173,7 +173,7 @@ export async function processGitHubRepo(options: ProcessGitHubRepoOptions): Prom token, owner: repo.owner, repo: repo.name, - title: "🤖 Add Copilot instructions via AgentRC", + title: "🤖 Add instructions via AgentRC", body: buildInstructionsPrBody(), head: branchName, base: repo.defaultBranch @@ -210,7 +210,7 @@ export async function processAzureRepo(options: ProcessAzureRepoOptions): Promis project: repo.project, repoId: repo.id, repoName: repo.name, - title: "🤖 Add Copilot instructions via AgentRC", + title: "🤖 Add instructions via AgentRC", body: buildInstructionsPrBody(), sourceBranch: branchName, targetBranch: repo.defaultBranch @@ -299,7 +299,7 @@ export type ProcessBatchReadinessRepoOptions = { }; /** - * Clone a single GitHub repo and run the AI-readiness report. + * Clone a single GitHub repo and run the readiness report. * Credential-stripped clone URL is restored on the remote after cloning. * Extracted from BatchReadinessTui so the TUI remains a thin presenter. */ diff --git a/packages/core/src/services/instructions.ts b/packages/core/src/services/instructions.ts index e0dde3f..d2ded05 100644 --- a/packages/core/src/services/instructions.ts +++ b/packages/core/src/services/instructions.ts @@ -430,8 +430,8 @@ export async function generateAreaInstructions( const preferredModel = options.model ?? DEFAULT_MODEL; const areaSystemContent = hasExistingInstructions - ? `You are an expert codebase analyst. Your task is to generate a concise .instructions.md file for a specific area of a codebase. This file will be used as a file-based custom instruction in VS Code Copilot, automatically applied when working on files matching certain patterns. This file should complement, not duplicate, existing instruction files. Use the Explore subagents and read-only tools to explore the codebase. When done, call the emit_file_content tool with the final markdown.` - : `You are an expert codebase analyst. Your task is to generate a concise .instructions.md file for a specific area of a codebase. This file will be used as a file-based custom instruction in VS Code Copilot, automatically applied when working on files matching certain patterns. Use the Explore subagents and read-only tools to explore the codebase. When done, call the emit_file_content tool with the final markdown.`; + ? `You are an expert codebase analyst. Your task is to generate a concise .instructions.md file for a specific area of a codebase. This file will be used as an area instruction in VS Code, automatically applied when working on files matching certain patterns. This file should complement, not duplicate, existing instruction files. Use the Explore subagents and read-only tools to explore the codebase. When done, call the emit_file_content tool with the final markdown.` + : `You are an expert codebase analyst. Your task is to generate a concise .instructions.md file for a specific area of a codebase. This file will be used as an area instruction in VS Code, automatically applied when working on files matching certain patterns. Use the Explore subagents and read-only tools to explore the codebase. When done, call the emit_file_content tool with the final markdown.`; const { tool: emitTool, getContent } = await createEmitTool(); @@ -470,7 +470,7 @@ export async function generateAreaInstructions( } }); - const prompt = `Analyze the "${area.name}" area of this codebase and generate a file-based instruction file. + const prompt = `Analyze the "${area.name}" area of this codebase and generate an area instruction file. This area covers files matching: ${applyToStr} ${area.description ? `Description: ${area.description}` : ""} diff --git a/packages/core/src/services/readiness.ts b/packages/core/src/services/readiness.ts index bdd8f9e..2932028 100644 --- a/packages/core/src/services/readiness.ts +++ b/packages/core/src/services/readiness.ts @@ -66,7 +66,7 @@ export function getLevelName(level: number): string { export function getLevelDescription(level: number): string { const descriptions: Record = { 1: "Repo builds, tests run, and basic tooling (linter, lockfile) is in place. AI agents can clone and get started.", - 2: "README, CONTRIBUTING guide, and custom AI instructions exist. Agents understand project context and conventions.", + 2: "README, CONTRIBUTING guide, and custom instructions exist. Agents understand project context and conventions.", 3: "CI/CD, security policies, CODEOWNERS, and observability are configured. Agents operate within well-defined guardrails.", 4: "MCP servers, custom agents, and AI skills are set up. Agents have deep integration with project-specific tools and workflows.", 5: "Full AI-native development: agents can independently plan, implement, test, and ship changes with minimal human oversight." @@ -683,7 +683,7 @@ export function buildCriteria(): ReadinessCriterion[] { }, { id: "custom-instructions", - title: "Custom AI instructions or agent guidance", + title: "Custom instructions or agent guidance", pillar: "ai-tooling", level: 1, scope: "repo", @@ -695,7 +695,7 @@ export function buildCriteria(): ReadinessCriterion[] { return { status: "fail", reason: - "Missing custom AI instructions (e.g. copilot-instructions.md, CLAUDE.md, AGENTS.md, .cursorrules).", + "Missing custom instructions (e.g. copilot-instructions.md, CLAUDE.md, AGENTS.md, .cursorrules).", evidence: [ "copilot-instructions.md", "CLAUDE.md", @@ -706,7 +706,7 @@ export function buildCriteria(): ReadinessCriterion[] { }; } - // Check for file-based instructions (.github/instructions/*.instructions.md) + // Check for area instructions (.github/instructions/*.instructions.md) const fileBasedInstructions = await hasFileBasedInstructions(context.repoPath); const areas = context.analysis.areas ?? []; @@ -715,13 +715,13 @@ export function buildCriteria(): ReadinessCriterion[] { if (fileBasedInstructions.length === 0) { return { status: "pass", - reason: `Root instructions found, but no file-based instructions for ${areas.length} detected areas. Run \`agentrc instructions --areas\` to generate.`, + reason: `Root instructions found, but no area instructions for ${areas.length} detected areas. Run \`agentrc instructions --areas\` to generate.`, evidence: [...rootFound, ...areas.map((a) => `${a.name}: missing .instructions.md`)] }; } return { status: "pass", - reason: `Root + ${fileBasedInstructions.length} file-based instruction(s) found.`, + reason: `Root + ${fileBasedInstructions.length} area instruction(s) found.`, evidence: [...rootFound, ...fileBasedInstructions] }; } diff --git a/packages/core/src/services/readiness/criteria.ts b/packages/core/src/services/readiness/criteria.ts index af6dfac..4043d21 100644 --- a/packages/core/src/services/readiness/criteria.ts +++ b/packages/core/src/services/readiness/criteria.ts @@ -278,7 +278,7 @@ export function buildCriteria(): ReadinessCriterion[] { }, { id: "custom-instructions", - title: "Custom AI instructions or agent guidance", + title: "Custom instructions or agent guidance", pillar: "ai-tooling", level: 1, scope: "repo", @@ -290,7 +290,7 @@ export function buildCriteria(): ReadinessCriterion[] { return { status: "fail", reason: - "Missing custom AI instructions (e.g. copilot-instructions.md, CLAUDE.md, AGENTS.md, .cursorrules).", + "Missing custom instructions (e.g. copilot-instructions.md, CLAUDE.md, AGENTS.md, .cursorrules).", evidence: [ "copilot-instructions.md", "CLAUDE.md", @@ -301,7 +301,7 @@ export function buildCriteria(): ReadinessCriterion[] { }; } - // Check for file-based instructions (.github/instructions/*.instructions.md) + // Check for area instructions (.github/instructions/*.instructions.md) const fileBasedInstructions = await hasFileBasedInstructions(context.repoPath); const areas = context.analysis.areas ?? []; @@ -310,13 +310,13 @@ export function buildCriteria(): ReadinessCriterion[] { if (fileBasedInstructions.length === 0) { return { status: "pass", - reason: `Root instructions found, but no file-based instructions for ${areas.length} detected areas. Run \`agentrc instructions --areas\` to generate.`, + reason: `Root instructions found, but no area instructions for ${areas.length} detected areas. Run \`agentrc instructions --areas\` to generate.`, evidence: [...rootFound, ...areas.map((a) => `${a.name}: missing .instructions.md`)] }; } return { status: "pass", - reason: `Root + ${fileBasedInstructions.length} file-based instruction(s) found.`, + reason: `Root + ${fileBasedInstructions.length} area instruction(s) found.`, evidence: [...rootFound, ...fileBasedInstructions] }; } diff --git a/packages/core/src/services/readiness/scoring.ts b/packages/core/src/services/readiness/scoring.ts index 3dd3bd6..bf79454 100644 --- a/packages/core/src/services/readiness/scoring.ts +++ b/packages/core/src/services/readiness/scoring.ts @@ -32,7 +32,7 @@ export function getLevelName(level: number): string { export function getLevelDescription(level: number): string { const descriptions: Record = { 1: "Repo builds, tests run, and basic tooling (linter, lockfile) is in place. AI agents can clone and get started.", - 2: "README, CONTRIBUTING guide, and custom AI instructions exist. Agents understand project context and conventions.", + 2: "README, CONTRIBUTING guide, and custom instructions exist. Agents understand project context and conventions.", 3: "CI/CD, security policies, CODEOWNERS, and observability are configured. Agents operate within well-defined guardrails.", 4: "MCP servers, custom agents, and AI skills are set up. Agents have deep integration with project-specific tools and workflows.", 5: "Full AI-native development: agents can independently plan, implement, test, and ship changes with minimal human oversight." diff --git a/packages/core/src/services/visualReport.ts b/packages/core/src/services/visualReport.ts index 05340dc..e7500cd 100644 --- a/packages/core/src/services/visualReport.ts +++ b/packages/core/src/services/visualReport.ts @@ -8,11 +8,7 @@ type VisualReportOptions = { }; export function generateVisualReport(options: VisualReportOptions): string { - const { - reports, - title = "AI Readiness Report", - generatedAt = new Date().toISOString() - } = options; + const { reports, title = "Readiness Report", generatedAt = new Date().toISOString() } = options; const successfulReports = reports.filter((r) => !r.error); const failedReports = reports.filter((r) => r.error); @@ -610,7 +606,7 @@ export function generateVisualReport(options: VisualReportOptions): string { }