diff --git a/cli/src/audit_schema.rs b/cli/src/audit_schema.rs new file mode 100644 index 0000000..b3a1f48 --- /dev/null +++ b/cli/src/audit_schema.rs @@ -0,0 +1,281 @@ +//! JSON Schema validation for audit output frontmatter. +//! +//! The schema is shipped at +//! `/.devtrail/schemas/audit-output.schema.v0.json` and validates +//! the YAML frontmatter of the markdown files produced by auditors and the +//! calibrator-reconciler during a `devtrail charter audit` cycle. The schema +//! uses `oneOf` to discriminate auditor outputs (primary/secondary) from +//! calibrator outputs via the `audit_role` field. +//! +//! Mirrors `charter_schema.rs` and `telemetry_schema.rs` — same shape, same +//! `ValidationIssue` integration so the existing output formatter handles +//! audit-output errors uniformly. + +use anyhow::{anyhow, Context, Result}; +use jsonschema::JSONSchema; +use serde_json::Value; +use std::path::{Path, PathBuf}; + +use crate::charter_schema::yaml_to_json_value; +use crate::validation::{Severity, ValidationIssue}; + +/// Path to the audit-output schema relative to a project's `.devtrail/`. +pub const SCHEMA_RELATIVE_PATH: &str = "schemas/audit-output.schema.v0.json"; + +/// A loaded and compiled audit-output schema, ready to validate YAML. +pub struct AuditOutputSchema { + compiled: JSONSchema, +} + +impl AuditOutputSchema { + /// Load and compile the audit-output schema from a project's `.devtrail/` + /// directory. Returns an error if the schema file is missing, not valid + /// JSON, or not a valid JSON Schema. + pub fn load(devtrail_dir: &Path) -> Result { + let path = devtrail_dir.join(SCHEMA_RELATIVE_PATH); + let raw = std::fs::read_to_string(&path).with_context(|| { + format!( + "Failed to read audit-output schema at {}. Run `devtrail repair` to restore framework files.", + path.display() + ) + })?; + Self::from_json_str(&raw, path) + } + + /// Compile the schema from a raw JSON string. Split out for testability. + pub fn from_json_str(raw: &str, source_path: PathBuf) -> Result { + let schema_json: Value = serde_json::from_str(raw).with_context(|| { + format!( + "Audit-output schema at {} is not valid JSON", + source_path.display() + ) + })?; + let compiled = JSONSchema::options() + .compile(&schema_json) + .map_err(|e| anyhow!("Failed to compile audit-output schema: {e}"))?; + Ok(Self { compiled }) + } + + /// Validate parsed audit-output frontmatter (YAML) against the schema. + /// Returns an empty Vec if valid; otherwise one `ValidationIssue` per + /// violation. + pub fn validate( + &self, + yaml_value: &serde_yaml::Value, + file_path: &Path, + ) -> Vec { + let json_value = match yaml_to_json_value(yaml_value) { + Ok(v) => v, + Err(e) => { + return vec![ValidationIssue { + file: file_path.to_path_buf(), + rule: "AUDIT-CONVERT".to_string(), + message: format!( + "Audit-output YAML cannot be converted to JSON for schema validation: {e}" + ), + severity: Severity::Error, + fix_hint: Some( + "Frontmatter must be JSON-compatible YAML (no tagged values or non-string keys)." + .to_string(), + ), + }]; + } + }; + let issues: Vec = match self.compiled.validate(&json_value) { + Ok(()) => Vec::new(), + Err(errors) => errors + .map(|err| ValidationIssue { + file: file_path.to_path_buf(), + rule: rule_from_error(&err), + message: format_message(&err), + severity: Severity::Error, + fix_hint: hint_for(&err), + }) + .collect(), + }; + issues + } +} + +fn rule_from_error(err: &jsonschema::ValidationError) -> String { + let path = err.schema_path.to_string(); + let trimmed = path.trim_start_matches('/').replace("/properties/", "/"); + if trimmed.is_empty() { + "AUDIT-SCHEMA".to_string() + } else { + format!("AUDIT-SCHEMA/{}", trimmed) + } +} + +fn format_message(err: &jsonschema::ValidationError) -> String { + let instance_path = err.instance_path.to_string(); + let location = if instance_path.is_empty() { + "frontmatter".to_string() + } else { + instance_path.trim_start_matches('/').replace('/', ".") + }; + format!("{} (at {})", err, location) +} + +fn hint_for(err: &jsonschema::ValidationError) -> Option { + let path = err.schema_path.to_string(); + if path.contains("/audit_role/enum") || path.contains("/audit_role/const") { + Some( + "audit_role must be one of: auditor-primary, auditor-secondary, calibrator-reconciler." + .to_string(), + ) + } else if path.contains("/charter_id/pattern") { + Some( + "charter_id must match CHARTER-NN[-slug] (e.g., CHARTER-05-baseline-recompute)." + .to_string(), + ) + } else if path.contains("/audited_at/format") || path.contains("/calibrated_at/format") { + Some("Date fields must be in YYYY-MM-DD format.".to_string()) + } else if path.contains("/auditors_reconciled/minItems") + || path.contains("/auditors_reconciled/maxItems") + { + Some( + "auditors_reconciled must have exactly 2 entries — the dual-audit pattern is fixed in v0." + .to_string(), + ) + } else if path.contains("/oneOf") { + Some( + "Output must match exactly one of the auditor or calibrator shapes — check audit_role and required fields." + .to_string(), + ) + } else if path.contains("/required") { + Some( + "Required field missing. See audit-output.schema.v0.json for the full list." + .to_string(), + ) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Minimal-shape schema for unit tests. The real schema lives at + /// dist/.devtrail/schemas/audit-output.schema.v0.json and is not bundled + /// into the binary — these tests are for the wrapper logic. + const TEST_SCHEMA: &str = r##"{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "oneOf": [ + { + "type": "object", + "required": ["audit_role", "auditor", "charter_id", "audited_at", "findings_total"], + "properties": { + "audit_role": { "type": "string", "enum": ["auditor-primary", "auditor-secondary"] }, + "auditor": { "type": "string", "minLength": 1 }, + "charter_id": { "type": "string", "pattern": "^CHARTER-[0-9]{2,}(-[a-z0-9-]+)?$" }, + "audited_at": { "type": "string", "format": "date" }, + "findings_total": { "type": "integer", "minimum": 0 } + } + }, + { + "type": "object", + "required": ["audit_role", "calibrator", "charter_id", "calibrated_at", "auditors_reconciled"], + "properties": { + "audit_role": { "type": "string", "const": "calibrator-reconciler" }, + "calibrator": { "type": "string", "minLength": 1 }, + "charter_id": { "type": "string", "pattern": "^CHARTER-[0-9]{2,}(-[a-z0-9-]+)?$" }, + "calibrated_at": { "type": "string", "format": "date" }, + "auditors_reconciled": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { "type": "string" } + } + } + } + ] + }"##; + + fn schema() -> AuditOutputSchema { + AuditOutputSchema::from_json_str(TEST_SCHEMA, PathBuf::from("test://schema")).unwrap() + } + + fn yaml(s: &str) -> serde_yaml::Value { + serde_yaml::from_str(s).unwrap() + } + + #[test] + fn valid_auditor_primary_passes() { + let v = yaml( + r#" +audit_role: auditor-primary +auditor: copilot-v1.0.37 +charter_id: CHARTER-05 +audited_at: "2026-05-03" +findings_total: 3 +"#, + ); + assert!(schema().validate(&v, Path::new("test.md")).is_empty()); + } + + #[test] + fn valid_calibrator_passes() { + let v = yaml( + r#" +audit_role: calibrator-reconciler +calibrator: claude-opus-4 +charter_id: CHARTER-05 +calibrated_at: "2026-05-03" +auditors_reconciled: + - auditor-primary.md + - auditor-secondary.md +"#, + ); + assert!(schema().validate(&v, Path::new("test.md")).is_empty()); + } + + #[test] + fn auditor_with_calibrator_role_fails() { + // audit_role: calibrator-reconciler but with auditor fields → fails oneOf. + let v = yaml( + r#" +audit_role: calibrator-reconciler +auditor: copilot-v1.0.37 +charter_id: CHARTER-05 +audited_at: "2026-05-03" +findings_total: 3 +"#, + ); + let issues = schema().validate(&v, Path::new("test.md")); + assert!(!issues.is_empty(), "expected oneOf violation, got no issues"); + } + + #[test] + fn calibrator_with_one_auditor_fails() { + let v = yaml( + r#" +audit_role: calibrator-reconciler +calibrator: claude-opus-4 +charter_id: CHARTER-05 +calibrated_at: "2026-05-03" +auditors_reconciled: + - auditor-primary.md +"#, + ); + let issues = schema().validate(&v, Path::new("test.md")); + assert!(!issues.is_empty(), "minItems:2 should be enforced"); + } + + #[test] + fn invalid_charter_id_pattern_caught() { + let v = yaml( + r#" +audit_role: auditor-primary +auditor: copilot +charter_id: not-a-charter-id +audited_at: "2026-05-03" +findings_total: 0 +"#, + ); + let issues = schema().validate(&v, Path::new("test.md")); + assert!(!issues.is_empty()); + } +} diff --git a/cli/src/commands/charter/audit.rs b/cli/src/commands/charter/audit.rs new file mode 100644 index 0000000..eb771ca --- /dev/null +++ b/cli/src/commands/charter/audit.rs @@ -0,0 +1,695 @@ +//! `devtrail charter audit` — orchestrate the dual-audit + calibrator cycle. +//! +//! Phase 3 v0 is **orchestration-only**: the command resolves prompts, awaits +//! the operator's auditor responses, validates outputs against the schema, +//! and prints the consolidated findings ready to paste into the Charter +//! telemetry. The CLI does NOT invoke any LLM API directly — the operator +//! runs the prompts in their auditor of choice (Copilot, Gemini, Claude, etc.) +//! and saves the responses to canonical paths. +//! +//! Three steps, each invokable independently: +//! +//! 1. **Prepare** (default invocation): resolve `auditor-primary.prompt.md` +//! and `auditor-secondary.prompt.md` against the Charter + git diff + +//! AILOGs, write them under `audit/charters//prompts/`. +//! 2. **Calibrate** (`--calibrate`): once both auditor responses exist at +//! `audit/charters//auditor-{primary,secondary}.md`, validate +//! them against the schema and resolve the calibrator prompt against +//! their findings. +//! 3. **Finalize** (`--finalize`): once the calibrator response exists, +//! validate everything, print a YAML-formatted `external_audit` block +//! for the operator to paste into the Charter telemetry, and print the +//! calibrator's reconciliation summary. +//! +//! Per RFC #82 the resolved prompts are persisted BEFORE any external +//! action. Per principle #10 (honesty about what the tool does not do) the +//! CLI does not pretend to talk to LLMs. + +use anyhow::{anyhow, bail, Context, Result}; +use colored::Colorize; +use std::path::{Path, PathBuf}; + +use crate::audit_schema::AuditOutputSchema; +use crate::charter::{self, Charter}; +use crate::utils; + +const DEFAULT_RANGE: &str = "HEAD~1..HEAD"; + +pub fn run( + path: &str, + charter_id: &str, + range: Option<&str>, + calibrate: bool, + finalize: bool, +) -> Result<()> { + if calibrate && finalize { + bail!("--calibrate and --finalize are mutually exclusive — run one at a time"); + } + + let resolved = utils::resolve_project_root(path) + .ok_or_else(|| anyhow!("DevTrail not installed. Run 'devtrail init' first."))?; + let project_root = &resolved.path; + let devtrail_dir = project_root.join(".devtrail"); + + // Resolve the Charter. + let (charters, _errors) = charter::discover_and_parse(project_root); + let charter = charter::find_by_id(&charters, charter_id) + .ok_or_else(|| anyhow!("Charter {} not found in docs/charters/", charter_id))? + .clone(); + + let canonical_id = canonical_charter_id(&charter.frontmatter.charter_id); + + let audit_dir = project_root + .join("audit") + .join("charters") + .join(&canonical_id); + let prompts_dir = audit_dir.join("prompts"); + utils::ensure_dir(&prompts_dir)?; + + let range = range.unwrap_or(DEFAULT_RANGE).to_string(); + + if finalize { + return run_finalize(project_root, &devtrail_dir, &audit_dir, &charter); + } + if calibrate { + return run_calibrate( + project_root, + &devtrail_dir, + &audit_dir, + &prompts_dir, + &charter, + &range, + ); + } + run_prepare( + project_root, + &devtrail_dir, + &audit_dir, + &prompts_dir, + &charter, + &range, + ) +} + +// ── Step 1: prepare ──────────────────────────────────────────────────────── + +fn run_prepare( + project_root: &Path, + devtrail_dir: &Path, + audit_dir: &Path, + prompts_dir: &Path, + charter: &Charter, + range: &str, +) -> Result<()> { + println!( + "{} {} ({})", + "Step 1/3:".cyan().bold(), + "PREPARE".bold(), + charter.frontmatter.charter_id.dimmed() + ); + + let context = build_audit_context(project_root, charter, range)?; + + for role in ["auditor-primary", "auditor-secondary"] { + let template_path = devtrail_dir + .join("audit-prompts") + .join(format!("{role}.md")); + let template = std::fs::read_to_string(&template_path).with_context(|| { + format!( + "Audit prompt template not found at {}. Run `devtrail repair` to restore framework files.", + template_path.display() + ) + })?; + let resolved = resolve_audit_template(&template, &context, role); + let out = prompts_dir.join(format!("{role}.prompt.md")); + std::fs::write(&out, resolved) + .with_context(|| format!("Failed to write resolved prompt to {}", out.display()))?; + println!( + " {} Wrote {}", + "✔".green().bold(), + relative_path(project_root, &out).display() + ); + } + + println!(); + println!(" {}", "Next:".bold()); + println!(" 1. Paste each prompt into your auditor of choice (use a model"); + println!(" of a different family per auditor — see CLI-REFERENCE)."); + println!(" 2. Save the auditor responses to:"); + println!( + " {}", + audit_dir + .join("auditor-primary.md") + .strip_prefix(project_root) + .unwrap_or_else(|_| audit_dir.as_ref()) + .display() + ); + println!( + " {}", + audit_dir + .join("auditor-secondary.md") + .strip_prefix(project_root) + .unwrap_or_else(|_| audit_dir.as_ref()) + .display() + ); + println!( + " 3. Run: {} {} --calibrate", + "devtrail charter audit".cyan(), + charter.frontmatter.charter_id.cyan() + ); + Ok(()) +} + +// ── Step 2: calibrate ────────────────────────────────────────────────────── + +fn run_calibrate( + project_root: &Path, + devtrail_dir: &Path, + audit_dir: &Path, + prompts_dir: &Path, + charter: &Charter, + range: &str, +) -> Result<()> { + println!( + "{} {} ({})", + "Step 2/3:".cyan().bold(), + "CALIBRATE".bold(), + charter.frontmatter.charter_id.dimmed() + ); + + let primary_path = audit_dir.join("auditor-primary.md"); + let secondary_path = audit_dir.join("auditor-secondary.md"); + + for (role, path) in [ + ("auditor-primary", &primary_path), + ("auditor-secondary", &secondary_path), + ] { + if !path.exists() { + bail!( + "{} not found. Save the {} response to that path before running --calibrate.", + path.display(), + role + ); + } + } + + let schema = AuditOutputSchema::load(devtrail_dir)?; + for path in [&primary_path, &secondary_path] { + let raw = std::fs::read_to_string(path) + .with_context(|| format!("Failed to read {}", path.display()))?; + let frontmatter = parse_frontmatter(&raw) + .with_context(|| format!("Failed to parse frontmatter in {}", path.display()))?; + let issues = schema.validate(&frontmatter, path); + if !issues.is_empty() { + eprintln!( + "{} validation issues in {}:", + "error:".red().bold(), + path.display() + ); + for issue in &issues { + eprintln!(" - {} [{}]", issue.message, issue.rule); + if let Some(hint) = &issue.fix_hint { + eprintln!(" {} {}", "hint:".cyan(), hint); + } + } + bail!("auditor output failed schema validation"); + } + println!( + " {} Validated {}", + "✔".green().bold(), + relative_path(project_root, path).display() + ); + } + + let primary_body = std::fs::read_to_string(&primary_path)?; + let secondary_body = std::fs::read_to_string(&secondary_path)?; + + let mut context = build_audit_context(project_root, charter, range)?; + context.auditor_primary_findings = primary_body; + context.auditor_secondary_findings = secondary_body; + + let template_path = devtrail_dir + .join("audit-prompts") + .join("calibrator-reconciler.md"); + let template = std::fs::read_to_string(&template_path).with_context(|| { + format!( + "Calibrator prompt template not found at {}. Run `devtrail repair`.", + template_path.display() + ) + })?; + let resolved = resolve_audit_template(&template, &context, "calibrator-reconciler"); + let out = prompts_dir.join("calibrator-reconciler.prompt.md"); + std::fs::write(&out, resolved) + .with_context(|| format!("Failed to write {}", out.display()))?; + println!( + " {} Wrote {}", + "✔".green().bold(), + relative_path(project_root, &out).display() + ); + + println!(); + println!(" {}", "Next:".bold()); + println!( + " 1. Run the calibrator prompt in a model of your choice (calibrator may"); + println!(" be of any family per roadmap §5.2 — heterogeneity is for the"); + println!(" auditor pair, not the calibrator)."); + println!( + " 2. Save the response to: {}", + audit_dir + .join("calibrator-reconciler.md") + .strip_prefix(project_root) + .unwrap_or_else(|_| audit_dir.as_ref()) + .display() + ); + println!( + " 3. Run: {} {} --finalize", + "devtrail charter audit".cyan(), + charter.frontmatter.charter_id.cyan() + ); + Ok(()) +} + +// ── Step 3: finalize ─────────────────────────────────────────────────────── + +fn run_finalize( + project_root: &Path, + devtrail_dir: &Path, + audit_dir: &Path, + charter: &Charter, +) -> Result<()> { + println!( + "{} {} ({})", + "Step 3/3:".cyan().bold(), + "FINALIZE".bold(), + charter.frontmatter.charter_id.dimmed() + ); + + let primary_path = audit_dir.join("auditor-primary.md"); + let secondary_path = audit_dir.join("auditor-secondary.md"); + let calibrator_path = audit_dir.join("calibrator-reconciler.md"); + + for (label, path) in [ + ("auditor-primary", &primary_path), + ("auditor-secondary", &secondary_path), + ("calibrator-reconciler", &calibrator_path), + ] { + if !path.exists() { + bail!( + "{} not found. {} must exist before --finalize. \ + Re-run --calibrate if the calibrator step is incomplete.", + path.display(), + label + ); + } + } + + let schema = AuditOutputSchema::load(devtrail_dir)?; + let mut auditor_summaries: Vec = Vec::new(); + for path in [&primary_path, &secondary_path] { + let raw = std::fs::read_to_string(path)?; + let fm = parse_frontmatter(&raw)?; + let issues = schema.validate(&fm, path); + if !issues.is_empty() { + eprintln!("{} {} failed schema validation", "error:".red().bold(), path.display()); + for issue in &issues { + eprintln!(" - {}", issue.message); + } + bail!("auditor output failed schema validation"); + } + let summary = AuditorSummary::from_frontmatter(&fm)?; + println!( + " {} Validated {} ({} findings, prompt: {})", + "✔".green().bold(), + relative_path(project_root, path).display(), + summary.findings_total, + summary.prompt_used.dimmed() + ); + auditor_summaries.push(summary); + } + + let calibrator_raw = std::fs::read_to_string(&calibrator_path)?; + let calibrator_fm = parse_frontmatter(&calibrator_raw)?; + let issues = schema.validate(&calibrator_fm, &calibrator_path); + if !issues.is_empty() { + eprintln!("{} calibrator failed schema validation", "error:".red().bold()); + for issue in &issues { + eprintln!(" - {}", issue.message); + } + bail!("calibrator output failed schema validation"); + } + println!( + " {} Validated {}", + "✔".green().bold(), + relative_path(project_root, &calibrator_path).display() + ); + + println!(); + println!(" {}", "Charter audit complete.".green().bold()); + println!(); + println!(" {}", "external_audit YAML — paste into telemetry:".bold()); + println!(" {}", "(charter_telemetry.external_audit array)".dimmed()); + println!(); + println!("{}", render_external_audit_yaml(&auditor_summaries)); + println!(); + println!(" {}", "Calibrator summary (copy to outcome.scope_change_notes if relevant):".dimmed()); + println!( + " {}", + relative_path(project_root, &calibrator_path).display().to_string().dimmed() + ); + Ok(()) +} + +// ── Audit context + template resolution ──────────────────────────────────── + +struct AuditContext { + charter_id: String, + charter_title: String, + charter_path: String, + charter_content: String, + git_range: String, + git_diff: String, + ailog_paths: String, + ailog_contents: String, + schema_path: String, + auditor_primary_findings: String, + auditor_secondary_findings: String, +} + +fn build_audit_context( + project_root: &Path, + charter: &Charter, + range: &str, +) -> Result { + let charter_content = std::fs::read_to_string(&charter.path) + .with_context(|| format!("Failed to read {}", charter.path.display()))?; + let charter_path_rel = relative_path(project_root, &charter.path) + .display() + .to_string(); + + let (ailog_paths, ailog_contents) = read_originating_ailogs(project_root, charter)?; + let git_diff = run_git_diff(project_root, range)?; + + Ok(AuditContext { + charter_id: charter.frontmatter.charter_id.clone(), + charter_title: charter::display_title(charter), + charter_path: charter_path_rel, + charter_content, + git_range: range.to_string(), + git_diff, + ailog_paths, + ailog_contents, + schema_path: ".devtrail/schemas/audit-output.schema.v0.json".to_string(), + auditor_primary_findings: String::new(), + auditor_secondary_findings: String::new(), + }) +} + +fn read_originating_ailogs(project_root: &Path, charter: &Charter) -> Result<(String, String)> { + let ailog_ids = match &charter.frontmatter.originating_ailogs { + Some(ids) if !ids.is_empty() => ids.clone(), + _ => return Ok(("(none)".to_string(), "(none)".to_string())), + }; + let agent_logs = project_root + .join(".devtrail") + .join("07-ai-audit") + .join("agent-logs"); + let mut paths = Vec::new(); + let mut contents = String::new(); + for id in &ailog_ids { + let prefix = id.split('-').take(5).collect::>().join("-"); + if let Some(found) = walk_for_prefix(&agent_logs, &prefix) { + paths.push( + relative_path(project_root, &found) + .display() + .to_string(), + ); + if let Ok(body) = std::fs::read_to_string(&found) { + contents.push_str(&format!("--- {} ---\n", id)); + contents.push_str(&body); + contents.push('\n'); + } + } else { + paths.push(format!("{} (NOT FOUND)", id)); + } + } + Ok((paths.join("\n"), contents)) +} + +fn walk_for_prefix(dir: &Path, prefix: &str) -> Option { + let entries = std::fs::read_dir(dir).ok()?; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + if let Some(found) = walk_for_prefix(&path, prefix) { + return Some(found); + } + continue; + } + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name.starts_with(prefix) && name.ends_with(".md") { + return Some(path); + } + } + } + None +} + +fn run_git_diff(project_root: &Path, range: &str) -> Result { + let output = std::process::Command::new("git") + .args(["diff", range]) + .current_dir(project_root) + .output() + .with_context(|| format!("Failed to invoke git diff {range}"))?; + if !output.status.success() { + let err = String::from_utf8_lossy(&output.stderr); + bail!("git diff {range} failed: {err}"); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +/// Substitute `{{placeholder}}` tokens in `template` with values from `ctx`. +/// `audit_role` is overridden per-call so the same context can be used for +/// primary, secondary, and calibrator passes. +fn resolve_audit_template(template: &str, ctx: &AuditContext, audit_role: &str) -> String { + let pairs: &[(&str, &str)] = &[ + ("{{charter_id}}", &ctx.charter_id), + ("{{charter_title}}", &ctx.charter_title), + ("{{charter_path}}", &ctx.charter_path), + ("{{charter_content}}", &ctx.charter_content), + ("{{git_range}}", &ctx.git_range), + ("{{git_diff}}", &ctx.git_diff), + ("{{ailog_paths}}", &ctx.ailog_paths), + ("{{ailog_contents}}", &ctx.ailog_contents), + ("{{audit_role}}", audit_role), + ("{{schema_path}}", &ctx.schema_path), + ("{{auditor_primary_findings}}", &ctx.auditor_primary_findings), + ( + "{{auditor_secondary_findings}}", + &ctx.auditor_secondary_findings, + ), + ]; + let mut out = template.to_string(); + for (placeholder, value) in pairs { + out = out.replace(placeholder, value); + } + out +} + +// ── Frontmatter parsing + auditor summary ────────────────────────────────── + +fn parse_frontmatter(raw: &str) -> Result { + let trimmed = raw.trim_start_matches('\u{feff}'); + let after = trimmed + .strip_prefix("---\n") + .ok_or_else(|| anyhow!("audit output does not start with `---` frontmatter delimiter"))?; + let end = after + .find("\n---") + .ok_or_else(|| anyhow!("frontmatter is not terminated by `---`"))?; + let yaml_str = &after[..end]; + Ok(serde_yaml::from_str(yaml_str)?) +} + +struct AuditorSummary { + auditor: String, + findings_total: u64, + findings_by_category: std::collections::BTreeMap, + audit_quality: Option, + prompt_used: String, +} + +impl AuditorSummary { + fn from_frontmatter(fm: &serde_yaml::Value) -> Result { + let auditor = fm + .get("auditor") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow!("auditor field missing"))? + .to_string(); + let findings_total = fm + .get("findings_total") + .and_then(|v| v.as_u64()) + .ok_or_else(|| anyhow!("findings_total missing"))?; + let findings_by_category = match fm.get("findings_by_category").and_then(|v| v.as_mapping()) + { + Some(map) => map + .iter() + .filter_map(|(k, v)| { + Some((k.as_str()?.to_string(), v.as_u64().unwrap_or(0))) + }) + .collect(), + None => Default::default(), + }; + let audit_quality = fm + .get("audit_quality") + .and_then(|v| v.as_str()) + .map(String::from); + let prompt_used = fm + .get("prompt_used") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + Ok(Self { + auditor, + findings_total, + findings_by_category, + audit_quality, + prompt_used, + }) + } +} + +fn render_external_audit_yaml(summaries: &[AuditorSummary]) -> String { + let mut out = String::new(); + for s in summaries { + out.push_str(&format!(" - auditor: \"{}\"\n", s.auditor)); + out.push_str(&format!(" findings_total: {}\n", s.findings_total)); + out.push_str(" findings_by_category:\n"); + for cat in [ + "hallucination", + "implementation_gap", + "real_debt", + "false_positive", + ] { + let count = s.findings_by_category.get(cat).copied().unwrap_or(0); + out.push_str(&format!(" {}: {}\n", cat, count)); + } + if let Some(quality) = &s.audit_quality { + out.push_str(&format!(" audit_quality: \"{}\"\n", quality)); + } + out.push_str(&format!( + " audit_notes: \"see audit/charters/{}/{}.md\"\n", + "", + if s.auditor.contains("primary") || s.findings_total > 0 { + "auditor-primary" + } else { + "auditor-secondary" + } + )); + } + out +} + +// ── Helpers ──────────────────────────────────────────────────────────────── + +fn canonical_charter_id(charter_id: &str) -> String { + // CHARTER-NN[-slug] → CHARTER-NN. + charter_id + .split_once('-') + .and_then(|(prefix, rest)| Some(format!("{}-{}", prefix, rest.split('-').next()?))) + .unwrap_or_else(|| charter_id.to_string()) +} + +fn relative_path(project_root: &Path, path: &Path) -> PathBuf { + path.strip_prefix(project_root) + .map(|p| p.to_path_buf()) + .unwrap_or_else(|_| path.to_path_buf()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonical_id_strips_slug() { + assert_eq!(canonical_charter_id("CHARTER-05"), "CHARTER-05"); + assert_eq!(canonical_charter_id("CHARTER-05-baseline"), "CHARTER-05"); + assert_eq!( + canonical_charter_id("CHARTER-12-batching-listtimeseries-04-f3"), + "CHARTER-12" + ); + } + + #[test] + fn resolve_template_substitutes_known_placeholders() { + let template = "id: {{charter_id}}\nrole: {{audit_role}}\nrange: {{git_range}}\n"; + let ctx = AuditContext { + charter_id: "CHARTER-01".into(), + charter_title: "T".into(), + charter_path: "p".into(), + charter_content: "c".into(), + git_range: "HEAD~1..HEAD".into(), + git_diff: "d".into(), + ailog_paths: "(none)".into(), + ailog_contents: "(none)".into(), + schema_path: "s".into(), + auditor_primary_findings: String::new(), + auditor_secondary_findings: String::new(), + }; + let out = resolve_audit_template(template, &ctx, "auditor-primary"); + assert_eq!( + out, + "id: CHARTER-01\nrole: auditor-primary\nrange: HEAD~1..HEAD\n" + ); + } + + #[test] + fn resolve_template_leaves_unknown_placeholders_intact() { + // If a template uses {{foo}} that isn't in our list, it stays as-is. + let template = "{{charter_id}} -- {{unknown_token}}"; + let ctx = AuditContext { + charter_id: "CHARTER-01".into(), + charter_title: "".into(), + charter_path: "".into(), + charter_content: "".into(), + git_range: "".into(), + git_diff: "".into(), + ailog_paths: "".into(), + ailog_contents: "".into(), + schema_path: "".into(), + auditor_primary_findings: String::new(), + auditor_secondary_findings: String::new(), + }; + let out = resolve_audit_template(template, &ctx, "x"); + assert_eq!(out, "CHARTER-01 -- {{unknown_token}}"); + } + + #[test] + fn parse_frontmatter_extracts_yaml_block() { + let raw = "---\nfoo: bar\nlist:\n - 1\n---\n\nbody\n"; + let v = parse_frontmatter(raw).unwrap(); + assert_eq!(v.get("foo").and_then(|v| v.as_str()), Some("bar")); + } + + #[test] + fn auditor_summary_extracts_fields() { + let yaml: serde_yaml::Value = serde_yaml::from_str( + r#" +auditor: copilot-v1.0.37 +findings_total: 5 +findings_by_category: + hallucination: 0 + implementation_gap: 2 + real_debt: 2 + false_positive: 1 +audit_quality: high +prompt_used: prompts/auditor-primary.prompt.md +"#, + ) + .unwrap(); + let s = AuditorSummary::from_frontmatter(&yaml).unwrap(); + assert_eq!(s.auditor, "copilot-v1.0.37"); + assert_eq!(s.findings_total, 5); + assert_eq!(s.findings_by_category.get("implementation_gap"), Some(&2)); + assert_eq!(s.audit_quality.as_deref(), Some("high")); + assert_eq!(s.prompt_used, "prompts/auditor-primary.prompt.md"); + } +} diff --git a/cli/src/commands/charter/mod.rs b/cli/src/commands/charter/mod.rs index 05ed796..b3509cc 100644 --- a/cli/src/commands/charter/mod.rs +++ b/cli/src/commands/charter/mod.rs @@ -6,6 +6,7 @@ //! drift check, in PR 3). //! Phase 3 will add `audit` (multi-model external audit). +pub mod audit; pub mod close; pub mod drift; pub mod list; diff --git a/cli/src/main.rs b/cli/src/main.rs index 6142930..7afafbc 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -4,6 +4,7 @@ use colored::Colorize; #[cfg(feature = "analyze")] mod analysis_engine; mod audit_engine; +mod audit_schema; mod charter; mod charter_schema; mod commands; @@ -300,6 +301,31 @@ enum CharterCommands { #[arg(long = "path", default_value = ".")] path: String, }, + /// Orchestrate a multi-model external audit cycle (3-step: prepare, + /// calibrate, finalize). Phase 3 v0 is orchestration-only — the CLI + /// resolves prompts, validates auditor outputs, and prints findings + /// for telemetry. It does NOT invoke LLM APIs; the operator runs the + /// prompts in their auditor of choice (Copilot, Gemini, Claude, etc.) + /// and saves responses to canonical paths. + Audit { + /// Charter identifier (CHARTER-NN, CHARTER-NN-slug, or just NN) + charter_id: String, + /// Git revision range (default: HEAD~1..HEAD) + #[arg(long)] + range: Option, + /// Step 2: read both auditor outputs from + /// audit/charters/CHARTER-NN/ and resolve the calibrator prompt. + #[arg(long, conflicts_with = "finalize")] + calibrate: bool, + /// Step 3: validate all 3 outputs against the schema and print the + /// external_audit YAML block ready to paste into the Charter + /// telemetry. + #[arg(long, conflicts_with = "calibrate")] + finalize: bool, + /// Project directory (default: current directory) + #[arg(long = "path", default_value = ".")] + path: String, + }, /// Detect file-vs-commit drift at Charter close (declared-but-not-modified /// files; scope expansion). Suppresses alerts on paths already documented /// as risks in the Charter's originating AILOGs. @@ -440,6 +466,19 @@ fn main() { range.as_deref(), no_ailog_suppress, ), + CharterCommands::Audit { + charter_id, + range, + calibrate, + finalize, + path, + } => commands::charter::audit::run( + &path, + &charter_id, + range.as_deref(), + calibrate, + finalize, + ), }, }; diff --git a/cli/tests/charter_audit_test.rs b/cli/tests/charter_audit_test.rs new file mode 100644 index 0000000..a5a887f --- /dev/null +++ b/cli/tests/charter_audit_test.rs @@ -0,0 +1,415 @@ +//! Integration tests for `devtrail charter audit` (Phase 3 v0). + +use assert_cmd::Command; +use predicates::prelude::*; +use std::path::Path; +use std::process::Command as StdCommand; +use tempfile::TempDir; + +const AUDIT_PROMPT_PRIMARY: &str = include_str!( + "../../dist/.devtrail/audit-prompts/auditor-primary.md" +); +const AUDIT_PROMPT_SECONDARY: &str = include_str!( + "../../dist/.devtrail/audit-prompts/auditor-secondary.md" +); +const AUDIT_PROMPT_CALIBRATOR: &str = include_str!( + "../../dist/.devtrail/audit-prompts/calibrator-reconciler.md" +); +const AUDIT_OUTPUT_SCHEMA: &str = include_str!( + "../../dist/.devtrail/schemas/audit-output.schema.v0.json" +); + +fn bash_available() -> bool { + StdCommand::new("git") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn setup_devtrail(dir: &Path) { + let devtrail = dir.join(".devtrail"); + std::fs::create_dir_all(devtrail.join("audit-prompts")).unwrap(); + std::fs::create_dir_all(devtrail.join("schemas")).unwrap(); + std::fs::create_dir_all(devtrail.join("07-ai-audit/agent-logs")).unwrap(); + std::fs::create_dir_all(devtrail.join("templates")).unwrap(); + std::fs::write(devtrail.join("config.yml"), "language: en\n").unwrap(); + std::fs::write( + devtrail.join("audit-prompts/auditor-primary.md"), + AUDIT_PROMPT_PRIMARY, + ) + .unwrap(); + std::fs::write( + devtrail.join("audit-prompts/auditor-secondary.md"), + AUDIT_PROMPT_SECONDARY, + ) + .unwrap(); + std::fs::write( + devtrail.join("audit-prompts/calibrator-reconciler.md"), + AUDIT_PROMPT_CALIBRATOR, + ) + .unwrap(); + std::fs::write( + devtrail.join("schemas/audit-output.schema.v0.json"), + AUDIT_OUTPUT_SCHEMA, + ) + .unwrap(); +} + +fn write_charter(dir: &Path) { + let charters = dir.join("docs/charters"); + std::fs::create_dir_all(&charters).unwrap(); + let body = r#"--- +charter_id: CHARTER-01 +status: in-progress +effort_estimate: M +trigger: "test" +--- + +# Charter: Audit test + +## Files to modify + +| File | Change | +|---|---| +| `src/foo.rs` | edit | + +## Tasks +1. Run. +"#; + std::fs::write(charters.join("01-audit-test.md"), body).unwrap(); +} + +fn git(dir: &Path, args: &[&str]) { + let status = StdCommand::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.com") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.com") + .status() + .expect("git failed"); + assert!(status.success(), "git {} failed", args.join(" ")); +} + +fn init_repo_with_diff(dir: &Path) { + std::fs::create_dir_all(dir.join("src")).unwrap(); + std::fs::write(dir.join("src/foo.rs"), "// initial\n").unwrap(); + git(dir, &["init", "-q", "-b", "main"]); + git(dir, &["add", "."]); + git(dir, &["commit", "-q", "-m", "initial"]); + std::fs::write(dir.join("src/foo.rs"), "// edited\n").unwrap(); + git(dir, &["add", "."]); + git(dir, &["commit", "-q", "-m", "edit"]); +} + +#[test] +fn audit_requires_devtrail_installed() { + let dir = TempDir::new().unwrap(); + Command::cargo_bin("devtrail") + .unwrap() + .args(["charter", "audit", "CHARTER-01", "--path"]) + .arg(dir.path().to_str().unwrap()) + .assert() + .failure() + .stderr(predicate::str::contains("not installed")); +} + +#[test] +fn audit_unknown_charter_fails() { + let dir = TempDir::new().unwrap(); + setup_devtrail(dir.path()); + Command::cargo_bin("devtrail") + .unwrap() + .args(["charter", "audit", "CHARTER-99", "--path"]) + .arg(dir.path().to_str().unwrap()) + .assert() + .failure() + .stderr(predicate::str::contains("CHARTER-99 not found")); +} + +#[test] +fn audit_prepare_writes_resolved_prompts() { + if !bash_available() { + eprintln!("skipping: git not available"); + return; + } + let dir = TempDir::new().unwrap(); + setup_devtrail(dir.path()); + write_charter(dir.path()); + init_repo_with_diff(dir.path()); + + Command::cargo_bin("devtrail") + .unwrap() + .args(["charter", "audit", "CHARTER-01", "--path"]) + .arg(dir.path().to_str().unwrap()) + .assert() + .success() + .stdout(predicate::str::contains("PREPARE")) + .stdout(predicate::str::contains("auditor-primary.prompt.md")) + .stdout(predicate::str::contains("auditor-secondary.prompt.md")) + .stdout(predicate::str::contains("--calibrate")); + + let prompts = dir.path().join("audit/charters/CHARTER-01/prompts"); + let primary = std::fs::read_to_string(prompts.join("auditor-primary.prompt.md")).unwrap(); + // Placeholder substitution happened. + assert!(primary.contains("CHARTER-01")); + assert!(primary.contains("auditor-primary")); + assert!(primary.contains("docs/charters/01-audit-test.md")); + // Diff was inlined. + assert!(primary.contains("// edited") || primary.contains("// initial")); + // Unknown placeholder syntax is gone. + assert!(!primary.contains("{{charter_id}}")); + assert!(!primary.contains("{{git_diff}}")); + + let secondary = std::fs::read_to_string(prompts.join("auditor-secondary.prompt.md")).unwrap(); + assert!(secondary.contains("auditor-secondary")); +} + +#[test] +fn audit_calibrate_requires_auditor_outputs() { + if !bash_available() { + return; + } + let dir = TempDir::new().unwrap(); + setup_devtrail(dir.path()); + write_charter(dir.path()); + init_repo_with_diff(dir.path()); + + // Skip prepare; go directly to calibrate. + Command::cargo_bin("devtrail") + .unwrap() + .args(["charter", "audit", "CHARTER-01", "--calibrate", "--path"]) + .arg(dir.path().to_str().unwrap()) + .assert() + .failure() + .stderr(predicate::str::contains("auditor-primary.md")) + .stderr(predicate::str::contains("not found")); +} + +#[test] +fn audit_calibrate_validates_outputs_against_schema() { + if !bash_available() { + return; + } + let dir = TempDir::new().unwrap(); + setup_devtrail(dir.path()); + write_charter(dir.path()); + init_repo_with_diff(dir.path()); + + let audit_dir = dir.path().join("audit/charters/CHARTER-01"); + std::fs::create_dir_all(&audit_dir).unwrap(); + + // Write a malformed auditor-primary.md (missing required findings_total). + std::fs::write( + audit_dir.join("auditor-primary.md"), + r#"--- +audit_role: auditor-primary +auditor: copilot +charter_id: CHARTER-01 +audited_at: "2026-05-03" +prompt_used: prompts/auditor-primary.prompt.md +--- + +# bad +"#, + ) + .unwrap(); + std::fs::write( + audit_dir.join("auditor-secondary.md"), + r#"--- +audit_role: auditor-secondary +auditor: gemini +charter_id: CHARTER-01 +audited_at: "2026-05-03" +findings_total: 0 +findings_by_category: + hallucination: 0 + implementation_gap: 0 + real_debt: 0 + false_positive: 0 +prompt_used: prompts/auditor-secondary.prompt.md +--- + +# good +"#, + ) + .unwrap(); + + Command::cargo_bin("devtrail") + .unwrap() + .args(["charter", "audit", "CHARTER-01", "--calibrate", "--path"]) + .arg(dir.path().to_str().unwrap()) + .assert() + .failure() + .stderr(predicate::str::contains("schema validation")); +} + +#[test] +fn audit_full_three_step_cycle_succeeds() { + if !bash_available() { + return; + } + let dir = TempDir::new().unwrap(); + setup_devtrail(dir.path()); + write_charter(dir.path()); + init_repo_with_diff(dir.path()); + + // Step 1: prepare. + Command::cargo_bin("devtrail") + .unwrap() + .args(["charter", "audit", "CHARTER-01", "--path"]) + .arg(dir.path().to_str().unwrap()) + .assert() + .success(); + + // Simulate the operator pasting valid auditor responses. + let audit_dir = dir.path().join("audit/charters/CHARTER-01"); + std::fs::write( + audit_dir.join("auditor-primary.md"), + r#"--- +audit_role: auditor-primary +auditor: copilot-v1.0.37 +charter_id: CHARTER-01 +git_range: "HEAD~1..HEAD" +prompt_used: prompts/auditor-primary.prompt.md +audited_at: "2026-05-03" +findings_total: 2 +findings_by_category: + hallucination: 0 + implementation_gap: 1 + real_debt: 1 + false_positive: 0 +audit_quality: high +--- + +# Audit by copilot + +## Findings + +### F1 — minor gap — implementation_gap + +Body. + +### F2 — leak — real_debt + +Body. +"#, + ) + .unwrap(); + std::fs::write( + audit_dir.join("auditor-secondary.md"), + r#"--- +audit_role: auditor-secondary +auditor: gemini-cli-v1.5 +charter_id: CHARTER-01 +git_range: "HEAD~1..HEAD" +prompt_used: prompts/auditor-secondary.prompt.md +audited_at: "2026-05-03" +findings_total: 1 +findings_by_category: + hallucination: 0 + implementation_gap: 1 + real_debt: 0 + false_positive: 0 +audit_quality: medium +--- + +# Audit by gemini + +## Findings + +### F1 — overlapping gap — implementation_gap + +Body. +"#, + ) + .unwrap(); + + // Step 2: calibrate. + Command::cargo_bin("devtrail") + .unwrap() + .args(["charter", "audit", "CHARTER-01", "--calibrate", "--path"]) + .arg(dir.path().to_str().unwrap()) + .assert() + .success() + .stdout(predicate::str::contains("CALIBRATE")) + .stdout(predicate::str::contains("calibrator-reconciler.prompt.md")) + .stdout(predicate::str::contains("--finalize")); + + // The resolved calibrator prompt should embed both auditors' findings. + let cal = std::fs::read_to_string( + audit_dir.join("prompts/calibrator-reconciler.prompt.md"), + ) + .unwrap(); + assert!(cal.contains("calibrator-reconciler")); + assert!(cal.contains("copilot-v1.0.37"), "primary auditor body should be embedded"); + assert!(cal.contains("gemini-cli-v1.5"), "secondary auditor body should be embedded"); + + // Simulate calibrator response. + std::fs::write( + audit_dir.join("calibrator-reconciler.md"), + r#"--- +audit_role: calibrator-reconciler +calibrator: claude-opus-4 +charter_id: CHARTER-01 +git_range: "HEAD~1..HEAD" +prompt_used: prompts/calibrator-reconciler.prompt.md +calibrated_at: "2026-05-03" +auditors_reconciled: + - auditor-primary.md + - auditor-secondary.md +findings_consolidated: 2 +findings_by_status: + agreed: 1 + disputed: 0 + unique_primary: 1 + unique_secondary: 0 + rejected: 0 +--- + +# Calibration + +## Reconciliation summary + +Both auditors converged on the implementation_gap; primary added a real_debt +that secondary missed. +"#, + ) + .unwrap(); + + // Step 3: finalize. + Command::cargo_bin("devtrail") + .unwrap() + .args(["charter", "audit", "CHARTER-01", "--finalize", "--path"]) + .arg(dir.path().to_str().unwrap()) + .assert() + .success() + .stdout(predicate::str::contains("FINALIZE")) + .stdout(predicate::str::contains("Charter audit complete")) + .stdout(predicate::str::contains("external_audit YAML")) + // Both auditors appear in the rendered YAML. + .stdout(predicate::str::contains("copilot-v1.0.37")) + .stdout(predicate::str::contains("gemini-cli-v1.5")); +} + +#[test] +fn audit_calibrate_and_finalize_are_mutually_exclusive() { + let dir = TempDir::new().unwrap(); + setup_devtrail(dir.path()); + + Command::cargo_bin("devtrail") + .unwrap() + .args([ + "charter", + "audit", + "CHARTER-01", + "--calibrate", + "--finalize", + "--path", + ]) + .arg(dir.path().to_str().unwrap()) + .assert() + .failure(); +}