diff --git a/__tests__/unit/rivet-oracle.test.js b/__tests__/unit/rivet-oracle.test.js new file mode 100644 index 0000000..70efcd7 --- /dev/null +++ b/__tests__/unit/rivet-oracle.test.js @@ -0,0 +1,274 @@ +jest.mock('../../src/logger.js', () => { + const log = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }; + return { getLogger: () => log }; +}); + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + isRivetProject, + runRivetValidate, + runRivetImpact, + runRivetOracle +} from '../../src/rivet-oracle.js'; + +function makeRunnerOk(stdout) { + return jest.fn().mockResolvedValue({ stdout, stderr: '' }); +} + +function makeRunnerOkButExitNonZero(stdout) { + // Mimic execFile behaviour: when child exits non-zero, the promise rejects + // with an Error object that carries `stdout` and `code`. + return jest.fn().mockRejectedValue( + Object.assign(new Error('command exited 1'), { stdout, stderr: '', code: 1 }) + ); +} + +function makeRunnerSpawnFailure(message) { + return jest.fn().mockRejectedValue(new Error(message)); +} + +describe('isRivetProject', () => { + let tmpDir; + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rivet-test-')); + }); + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('returns true when rivet.yaml is at the root', () => { + fs.writeFileSync(path.join(tmpDir, 'rivet.yaml'), 'project: x\n'); + expect(isRivetProject(tmpDir)).toBe(true); + }); + + it('returns false when no rivet.yaml exists', () => { + expect(isRivetProject(tmpDir)).toBe(false); + }); + + it('returns false on a non-existent path', () => { + expect(isRivetProject('/nonexistent/path/12345')).toBe(false); + }); +}); + +describe('runRivetValidate', () => { + let tmpDir; + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rivet-test-')); + fs.writeFileSync(path.join(tmpDir, 'rivet.yaml'), 'project: x\n'); + }); + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + const sampleJson = JSON.stringify({ + command: 'validate', + result: 'FAIL', + errors: 2, + warnings: 1, + infos: 5, + diagnostics: [ + { artifact_id: 'REQ-001', message: 'missing required field x', severity: 'error' }, + { artifact_id: 'REQ-002', message: 'unrecognised field', severity: 'warning' }, + { artifact_id: 'REQ-099', message: 'should have decision link', severity: 'info' }, + { artifact_id: 'REQ-005', message: 'broken schema', severity: 'error' } + ], + lifecycle_gaps: [ + { artifact_id: 'REQ-100', type: 'requirement', reason: 'no downstream artifacts found' } + ], + broken_cross_refs: [ + { from: 'REQ-200', to: 'DEC-200', reason: 'target does not exist' } + ] + }); + + it('parses validate output and converts errors+warnings to findings (drops info)', async () => { + const result = await runRivetValidate('/usr/bin/rivet', tmpDir, { + runner: makeRunnerOk(sampleJson) + }); + expect(result.ok).toBe(true); + expect(result.result).toBe('FAIL'); + // 2 errors + 1 warning + 1 lifecycle_gap + 1 broken_cross_ref = 5 findings; + // info dropped. + expect(result.findings).toHaveLength(5); + expect(result.findings.every((f) => f.severity !== 'info')).toBe(true); + expect(result.findings.find((f) => f.artifact_id === 'REQ-099')).toBeUndefined(); + }); + + it('still parses JSON when CLI exits non-zero (validate FAIL is the common case)', async () => { + const result = await runRivetValidate('/usr/bin/rivet', tmpDir, { + runner: makeRunnerOkButExitNonZero(sampleJson) + }); + expect(result.ok).toBe(true); + expect(result.result).toBe('FAIL'); + expect(result.findings.length).toBeGreaterThan(0); + }); + + it('returns ok:false when CLI cannot be spawned', async () => { + const result = await runRivetValidate('/usr/bin/rivet', tmpDir, { + runner: makeRunnerSpawnFailure('ENOENT') + }); + expect(result.ok).toBe(false); + }); + + it('returns ok:false on unparseable stdout', async () => { + const result = await runRivetValidate('/usr/bin/rivet', tmpDir, { + runner: makeRunnerOk('this is not json') + }); + expect(result.ok).toBe(false); + expect(result.error).toMatch(/parseable JSON/); + }); + + it('rejects when target is not a rivet project', async () => { + fs.rmSync(path.join(tmpDir, 'rivet.yaml')); + const result = await runRivetValidate('/usr/bin/rivet', tmpDir, { + runner: makeRunnerOk(sampleJson) + }); + expect(result.ok).toBe(false); + expect(result.error).toMatch(/not a rivet project/); + }); + + it('every finding includes the artifact_id and a non-hedging claim', async () => { + const result = await runRivetValidate('/usr/bin/rivet', tmpDir, { + runner: makeRunnerOk(sampleJson) + }); + for (const f of result.findings) { + expect(typeof f.artifact_id).toBe('string'); + expect(f.artifact_id.length).toBeGreaterThan(0); + expect(f.claim).toContain(f.artifact_id); + // Sanity: must not contain hedging language by construction + expect(f.claim).not.toMatch(/might|could|may possibly/i); + } + }); +}); + +describe('runRivetImpact', () => { + let tmpDir; + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rivet-test-')); + fs.writeFileSync(path.join(tmpDir, 'rivet.yaml'), 'project: x\n'); + }); + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + const sampleJson = JSON.stringify({ + command: 'impact', + added: ['NEW-1', 'NEW-2'], + removed: ['OLD-1'], + changed: [{ id: 'CHG-1', summary: 'fields changed', title: 'Some artifact' }], + directly_affected: [ + { id: 'AFF-1', depth: 1, reason: ['-> issued-by X'], title: 'Affected one' } + ], + transitively_affected: [ + { id: 'TRA-1', depth: 2, reason: ['-> blah'], title: '' }, + { id: 'TRA-2', depth: 3, reason: ['-> blah'], title: '' } + ], + summary: { added: 2, removed: 1, changed: 1, direct: 1, transitive: 2 } + }); + + it('surfaces removed + directly_affected, summarises transitive', async () => { + const result = await runRivetImpact('/usr/bin/rivet', tmpDir, 'main', { + runner: makeRunnerOk(sampleJson) + }); + expect(result.ok).toBe(true); + + // 1 removed + 1 directly_affected + 1 transitive-summary = 3 findings. + // We do NOT surface 'added' or 'changed' as findings — those are diff + // metadata, not concerns. + expect(result.findings).toHaveLength(3); + + const byId = Object.fromEntries(result.findings.map((f) => [f.artifact_id, f])); + expect(byId['OLD-1']).toBeDefined(); + expect(byId['OLD-1'].claim).toMatch(/removed/i); + expect(byId['AFF-1']).toBeDefined(); + expect(byId['']).toBeDefined(); + expect(byId[''].claim).toMatch(/2 additional artifacts/); + }); + + it('omits transitive summary when transitive list is empty', async () => { + const empty = JSON.stringify({ + command: 'impact', + added: [], removed: [], changed: [], + directly_affected: [], + transitively_affected: [], + summary: { added: 0, removed: 0, changed: 0, direct: 0, transitive: 0 } + }); + const result = await runRivetImpact('/usr/bin/rivet', tmpDir, 'main', { + runner: makeRunnerOk(empty) + }); + expect(result.findings).toHaveLength(0); + }); +}); + +describe('runRivetOracle', () => { + let tmpDir; + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rivet-test-')); + fs.writeFileSync(path.join(tmpDir, 'rivet.yaml'), 'project: x\n'); + }); + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('returns applicable:false on non-rivet repo', async () => { + fs.rmSync(path.join(tmpDir, 'rivet.yaml')); + const r = await runRivetOracle('/usr/bin/rivet', tmpDir, {}); + expect(r.applicable).toBe(false); + }); + + it('runs validate only when no baseRef provided', async () => { + const validateJson = JSON.stringify({ + command: 'validate', result: 'PASS', + errors: 0, warnings: 0, infos: 0, + diagnostics: [], lifecycle_gaps: [], broken_cross_refs: [] + }); + const runner = makeRunnerOk(validateJson); + const r = await runRivetOracle('/usr/bin/rivet', tmpDir, { runner }); + expect(r.applicable).toBe(true); + expect(runner).toHaveBeenCalledTimes(1); + expect(r.impact).toBeNull(); + }); + + it('runs both validate and impact when baseRef provided', async () => { + const validateJson = JSON.stringify({ + command: 'validate', result: 'PASS', + errors: 0, warnings: 0, infos: 0, + diagnostics: [], lifecycle_gaps: [], broken_cross_refs: [] + }); + const impactJson = JSON.stringify({ + command: 'impact', + added: [], removed: [], changed: [], + directly_affected: [], transitively_affected: [], + summary: { added: 0, removed: 0, changed: 0, direct: 0, transitive: 0 } + }); + const runner = jest.fn() + .mockResolvedValueOnce({ stdout: validateJson, stderr: '' }) + .mockResolvedValueOnce({ stdout: impactJson, stderr: '' }); + const r = await runRivetOracle('/usr/bin/rivet', tmpDir, { + baseRef: 'main', + runner + }); + expect(r.applicable).toBe(true); + expect(runner).toHaveBeenCalledTimes(2); + expect(r.impact).not.toBeNull(); + }); + + it('returns ok:true even when one of the two oracles fails (degraded)', async () => { + const validateJson = JSON.stringify({ + command: 'validate', result: 'PASS', + errors: 0, warnings: 0, infos: 0, + diagnostics: [], lifecycle_gaps: [], broken_cross_refs: [] + }); + const runner = jest.fn() + .mockResolvedValueOnce({ stdout: validateJson, stderr: '' }) + .mockRejectedValueOnce(new Error('impact crashed')); + const r = await runRivetOracle('/usr/bin/rivet', tmpDir, { + baseRef: 'main', + runner + }); + expect(r.ok).toBe(true); + expect(r.validate.result).toBe('PASS'); + expect(r.impact.error).toMatch(/impact crashed/); + }); +}); diff --git a/src/rivet-oracle.js b/src/rivet-oracle.js new file mode 100644 index 0000000..bf960dc --- /dev/null +++ b/src/rivet-oracle.js @@ -0,0 +1,264 @@ +/** + * Rivet mechanical oracle. + * + * Runs the `rivet` CLI on a checked-out repo tree and converts its JSON + * output into Finding records the AI review pipeline can consume directly, + * bypassing the model. These findings are *already* mechanically validated: + * they cite real artifact IDs, were emitted by a deterministic tool, and + * cannot be hallucinated. + * + * Two oracle modes: + * - validate: schema + traceability check. Reports per-artifact errors, + * warnings, broken cross-refs, lifecycle gaps. + * - impact: diff against a baseline ref. Reports added/removed/changed + * artifacts plus the transitive closure of affected artifacts. + * + * The module deliberately does NOT clone the repo or install the binary — + * that's the caller's job. Keeping side effects out makes it trivially + * unit-testable with a fake `runner` injected in tests. + */ + +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import fs from 'node:fs'; +import path from 'node:path'; +import { getLogger } from './logger.js'; + +const execFileP = promisify(execFile); + +const DEFAULT_TIMEOUT_MS = 60 * 1000; + +/** Exit codes are not 0 for FAIL — but stdout still has the JSON. */ +async function runCli(binary, args, opts = {}) { + const { cwd, timeout = DEFAULT_TIMEOUT_MS, runner = execFileP } = opts; + try { + const { stdout } = await runner(binary, args, { cwd, timeout, maxBuffer: 16 * 1024 * 1024 }); + return { ok: true, stdout }; + } catch (err) { + // Non-zero exit (e.g. validate FAIL) is expected — stdout is still valid JSON. + if (err.stdout) return { ok: true, stdout: err.stdout, exitCode: err.code }; + return { ok: false, error: err.message, code: err.code }; + } +} + +function safeParseJson(text) { + try { + return JSON.parse(text); + } catch { + return null; + } +} + +/** + * Detect whether the given working tree is a rivet project. + * Checks for `rivet.yaml` at the root. + */ +export function isRivetProject(repoPath) { + return fs.existsSync(path.join(repoPath, 'rivet.yaml')); +} + +/** + * Convert a single rivet validate diagnostic into a Finding. + * Severity 'info' is dropped — too noisy for code review. + */ +function diagnosticToFinding(diag) { + if (!diag || diag.severity === 'info') return null; + return { + source: 'oracle:rivet-validate', + severity: diag.severity, + artifact_id: diag.artifact_id, + claim: `${diag.artifact_id}: ${diag.message}` + }; +} + +/** + * Convert lifecycle-coverage gaps. Each gap is an artifact missing + * downstream evidence — the slop signal we want to surface. + */ +function gapToFinding(gap) { + if (!gap || typeof gap !== 'object') return null; + const id = gap.artifact_id || gap.id; + if (!id) return null; + return { + source: 'oracle:rivet-validate', + severity: 'warning', + artifact_id: id, + claim: `${id} (${gap.type || 'artifact'}) — ${gap.reason || gap.message || 'lifecycle coverage gap: no downstream artifacts'}` + }; +} + +/** + * Convert broken cross-refs / circular deps into findings. + */ +function brokenCrossRefToFinding(ref) { + const from = ref?.from || ref?.source || 'unknown'; + const to = ref?.to || ref?.target || 'unknown'; + return { + source: 'oracle:rivet-validate', + severity: 'error', + artifact_id: from, + claim: `Broken cross-reference: ${from} → ${to}${ref?.reason ? ` (${ref.reason})` : ''}` + }; +} + +/** + * Run `rivet validate --format json` and convert diagnostics + gaps + broken + * refs to Finding records. Returns: + * + * { + * ok: true | false, + * result: 'PASS' | 'FAIL' | 'UNKNOWN', + * findings: Finding[], + * summary: { errors, warnings, ... }, + * error?: string // when ok = false + * } + */ +export async function runRivetValidate(binary, repoPath, opts = {}) { + if (!isRivetProject(repoPath)) { + return { ok: false, error: 'not a rivet project (no rivet.yaml at root)' }; + } + const cliResult = await runCli(binary, ['validate', '--format', 'json'], { + cwd: repoPath, + timeout: opts.timeout, + runner: opts.runner + }); + if (!cliResult.ok) { + getLogger().warn({ err: cliResult.error }, 'rivet validate failed to spawn'); + return { ok: false, error: cliResult.error || 'spawn failed' }; + } + const data = safeParseJson(cliResult.stdout); + if (!data) { + return { ok: false, error: 'rivet validate did not emit parseable JSON' }; + } + + const findings = []; + for (const d of data.diagnostics || []) { + const f = diagnosticToFinding(d); + if (f) findings.push(f); + } + for (const g of data.lifecycle_gaps || []) { + const f = gapToFinding(g); + if (f) findings.push(f); + } + for (const r of data.broken_cross_refs || []) { + findings.push(brokenCrossRefToFinding(r)); + } + + return { + ok: true, + result: data.result || 'UNKNOWN', + findings, + summary: { + errors: data.errors ?? 0, + warnings: data.warnings ?? 0, + infos: data.infos ?? 0, + broken_cross_refs: (data.broken_cross_refs || []).length, + circular_dependencies: (data.circular_dependencies || []).length, + lifecycle_gaps: (data.lifecycle_gaps || []).length + } + }; +} + +/** + * Run `rivet impact --since= --format json` and convert removed + + * transitively-affected artifacts into findings. We intentionally do NOT + * surface every `added` or `changed` artifact — those are diff metadata, + * not concerns. We DO surface: + * + * - removed artifacts (anything dropped from the trace graph is risky) + * - directly_affected (what this PR breaks the contract of) + * + * `transitively_affected` is included as a single summary finding when + * non-empty; full list is too noisy for a PR comment. + */ +export async function runRivetImpact(binary, repoPath, baseRef, opts = {}) { + if (!isRivetProject(repoPath)) { + return { ok: false, error: 'not a rivet project (no rivet.yaml at root)' }; + } + const cliResult = await runCli( + binary, + ['impact', '--since', baseRef, '--format', 'json'], + { cwd: repoPath, timeout: opts.timeout, runner: opts.runner } + ); + if (!cliResult.ok) { + return { ok: false, error: cliResult.error || 'spawn failed' }; + } + const data = safeParseJson(cliResult.stdout); + if (!data) { + return { ok: false, error: 'rivet impact did not emit parseable JSON' }; + } + + const findings = []; + + for (const id of data.removed || []) { + findings.push({ + source: 'oracle:rivet-impact', + severity: 'warning', + artifact_id: id, + claim: `Artifact removed: ${id}. Verify nothing downstream still depends on it.` + }); + } + + for (const item of data.directly_affected || []) { + const id = item?.id || 'unknown'; + const reason = Array.isArray(item?.reason) ? item.reason.join('; ') : ''; + const title = item?.title ? ` — ${item.title}` : ''; + findings.push({ + source: 'oracle:rivet-impact', + severity: 'info', + artifact_id: id, + claim: `Directly affected: ${id}${title}${reason ? ` (${reason})` : ''}.` + }); + } + + const transitiveCount = (data.transitively_affected || []).length; + if (transitiveCount > 0) { + findings.push({ + source: 'oracle:rivet-impact', + severity: 'info', + artifact_id: '', + claim: `${transitiveCount} additional artifacts transitively affected by this PR. Run \`rivet impact --since=${baseRef}\` for the full list.` + }); + } + + return { + ok: true, + findings, + summary: data.summary || { + added: (data.added || []).length, + removed: (data.removed || []).length, + changed: (data.changed || []).length, + direct: (data.directly_affected || []).length, + transitive: (data.transitively_affected || []).length + } + }; +} + +/** + * One-shot helper: run both validate and impact (when baseRef given), merge + * findings, return a single result. Errors from either are non-fatal — + * findings from the surviving call are still returned. + */ +export async function runRivetOracle(binary, repoPath, opts = {}) { + if (!isRivetProject(repoPath)) { + return { ok: false, applicable: false, error: 'not a rivet project' }; + } + + const validate = await runRivetValidate(binary, repoPath, opts); + let impact = null; + if (opts.baseRef) { + impact = await runRivetImpact(binary, repoPath, opts.baseRef, opts); + } + + const findings = []; + if (validate.ok) findings.push(...validate.findings); + if (impact?.ok) findings.push(...impact.findings); + + return { + ok: true, + applicable: true, + validate: validate.ok ? { result: validate.result, summary: validate.summary } : { error: validate.error }, + impact: impact ? (impact.ok ? { summary: impact.summary } : { error: impact.error }) : null, + findings + }; +}