Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 191 additions & 0 deletions packages/adf/src/__tests__/evidence.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
import { describe, it, expect } from 'vitest';
import { evaluateEvidence } from '../evidence';
import type { BundleResult } from '../types';

function makeBundleResult(overrides: Partial<BundleResult> = {}): BundleResult {
return {
manifest: {
version: '0.1',
defaultLoad: ['core.adf'],
onDemand: [],
rules: [],
sync: [],
cadence: [],
metrics: [],
},
resolvedModules: ['core.adf'],
mergedDocument: {
version: '0.1',
sections: [
{
key: 'CONSTRAINTS',
decoration: null,
content: { type: 'list', items: ['No secrets'] },
weight: 'load-bearing',
},
],
},
tokenEstimate: 50,
tokenBudget: 4000,
tokenUtilization: 0.0125,
perModuleTokens: { 'core.adf': 50 },
moduleBudgetOverruns: [],
triggerMatches: [],
unmatchedModules: [],
advisoryOnlyModules: [],
...overrides,
};
}

describe('evaluateEvidence', () => {
it('forwards constraint results from validateConstraints', () => {
const bundle = makeBundleResult({
mergedDocument: {
version: '0.1',
sections: [
{
key: 'METRICS',
decoration: null,
content: {
type: 'metric',
entries: [{ key: 'loc', value: 100, ceiling: 200, unit: 'lines' }],
},
weight: 'load-bearing',
},
],
},
});
const report = evaluateEvidence(bundle, { loc: 150 });
expect(report.constraints).toHaveLength(1);
expect(report.constraints[0].status).toBe('pass');
expect(report.constraints[0].value).toBe(150);
expect(report.allPassing).toBe(true);
});

it('reports failing constraints', () => {
const bundle = makeBundleResult({
mergedDocument: {
version: '0.1',
sections: [
{
key: 'METRICS',
decoration: null,
content: {
type: 'metric',
entries: [{ key: 'loc', value: 100, ceiling: 200, unit: 'lines' }],
},
},
],
},
});
const report = evaluateEvidence(bundle, { loc: 250 });
expect(report.constraints[0].status).toBe('fail');
expect(report.allPassing).toBe(false);
expect(report.failCount).toBe(1);
});

it('reports warn status at ceiling boundary', () => {
const bundle = makeBundleResult({
mergedDocument: {
version: '0.1',
sections: [
{
key: 'METRICS',
decoration: null,
content: {
type: 'metric',
entries: [{ key: 'loc', value: 100, ceiling: 200, unit: 'lines' }],
},
},
],
},
});
const report = evaluateEvidence(bundle, { loc: 200 });
expect(report.constraints[0].status).toBe('warn');
expect(report.warnCount).toBe(1);
});

it('forwards token data from bundle', () => {
const bundle = makeBundleResult({
tokenEstimate: 500,
tokenBudget: 4000,
tokenUtilization: 0.125,
perModuleTokens: { 'core.adf': 300, 'frontend.adf': 200 },
});
const report = evaluateEvidence(bundle);
expect(report.tokenEstimate).toBe(500);
expect(report.tokenBudget).toBe(4000);
expect(report.tokenUtilization).toBe(0.125);
expect(report.perModuleTokens).toEqual({ 'core.adf': 300, 'frontend.adf': 200 });
});

it('forwards module budget overruns', () => {
const overruns = [{ module: 'frontend.adf', tokens: 150, budget: 100 }];
const bundle = makeBundleResult({ moduleBudgetOverruns: overruns });
const report = evaluateEvidence(bundle);
expect(report.moduleBudgetOverruns).toEqual(overruns);
});

it('forwards advisory-only modules', () => {
const bundle = makeBundleResult({ advisoryOnlyModules: ['frontend.adf'] });
const report = evaluateEvidence(bundle);
expect(report.advisoryOnlyModules).toEqual(['frontend.adf']);
});

it('includes weight summary', () => {
const report = evaluateEvidence(makeBundleResult());
expect(report.weightSummary).toBeDefined();
expect(report.weightSummary.total).toBeGreaterThan(0);
});

it('detects stale baselines when context drifts beyond threshold', () => {
const bundle = makeBundleResult({
mergedDocument: {
version: '0.1',
sections: [
{
key: 'METRICS',
decoration: null,
content: {
type: 'metric',
entries: [{ key: 'loc', value: 100, ceiling: 200, unit: 'lines' }],
},
},
],
},
});
// current 150, baseline 100 → ratio 1.5 > default threshold 1.2
const report = evaluateEvidence(bundle, { loc: 150 });
expect(report.staleBaselines).toHaveLength(1);
expect(report.staleBaselines[0].metric).toBe('loc');
expect(report.staleBaselines[0].baseline).toBe(100);
expect(report.staleBaselines[0].current).toBe(150);
expect(report.staleBaselines[0].rationaleRequired).toBe(true);
});

it('respects custom stale threshold', () => {
const bundle = makeBundleResult({
mergedDocument: {
version: '0.1',
sections: [
{
key: 'METRICS',
decoration: null,
content: {
type: 'metric',
entries: [{ key: 'loc', value: 100, ceiling: 200, unit: 'lines' }],
},
},
],
},
});
// ratio 1.5, threshold 2.0 → not stale
const report = evaluateEvidence(bundle, { loc: 150 }, 2.0);
expect(report.staleBaselines).toHaveLength(0);
});

it('returns empty stale baselines when no context provided', () => {
const report = evaluateEvidence(makeBundleResult());
expect(report.staleBaselines).toEqual([]);
});
});
116 changes: 116 additions & 0 deletions packages/adf/src/evidence.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/**
* ADF Evidence — unified evaluation combining constraint validation,
* token analysis, and baseline staleness.
*
* Produces a single EvidenceReport from a BundleResult and optional
* external metric context, replacing the need for consumers to manually
* assemble evidence from multiple sources.
*/

import type {
AdfDocument,
BundleResult,
EvidenceResult,
} from './types';
import { validateConstraints } from './validator';

// ============================================================================
// Types
// ============================================================================

export interface StaleBaselineWarning {
metric: string;
baseline: number;
current: number;
delta: number;
ratio: number;
recommendedCeiling: number;
rationaleRequired: boolean;
}

export interface EvidenceReport extends EvidenceResult {
tokenEstimate: number;
tokenBudget: number | null;
tokenUtilization: number | null;
perModuleTokens: Record<string, number>;
moduleBudgetOverruns: BundleResult['moduleBudgetOverruns'];
advisoryOnlyModules: string[];
staleBaselines: StaleBaselineWarning[];
}

// ============================================================================
// Evaluation
// ============================================================================

/**
* Evaluate evidence from a bundle result and optional metric context.
*
* Combines constraint validation, token budget analysis, module budget
* overruns, advisory-only module warnings, and stale baseline detection
* into a single unified report.
*
* @param bundle - Result from bundleModules()
* @param context - Optional external metric overrides (e.g., actual LOC counts)
* @param staleThreshold - Ratio threshold for stale baseline detection (default 1.2)
*/
export function evaluateEvidence(
bundle: BundleResult,
context?: Record<string, number>,
staleThreshold?: number,
): EvidenceReport {
const evidence = validateConstraints(bundle.mergedDocument, context);
const staleBaselines = detectStaleBaselines(
bundle.mergedDocument,
context,
staleThreshold ?? 1.2,
);

return {
...evidence,
tokenEstimate: bundle.tokenEstimate,
tokenBudget: bundle.tokenBudget,
tokenUtilization: bundle.tokenUtilization,
perModuleTokens: bundle.perModuleTokens,
moduleBudgetOverruns: bundle.moduleBudgetOverruns,
advisoryOnlyModules: bundle.advisoryOnlyModules,
staleBaselines,
};
}

// ============================================================================
// Stale Baseline Detection
// ============================================================================

/**
* Detect metrics whose current values have drifted significantly from
* their ADF baselines, indicating the ceilings may need recalibration.
*/
function detectStaleBaselines(
doc: AdfDocument,
context: Record<string, number> | undefined,
staleThreshold: number,
): StaleBaselineWarning[] {
if (!context) return [];
const warnings: StaleBaselineWarning[] = [];
for (const section of doc.sections) {
if (section.key !== 'METRICS' || section.content.type !== 'metric') continue;
for (const entry of section.content.entries) {
if (entry.value <= 0) continue;
const key = entry.key.toLowerCase();
const current = context[key];
if (!Number.isFinite(current)) continue;
const ratio = current / entry.value;
if (ratio < staleThreshold) continue;
warnings.push({
metric: key,
baseline: entry.value,
current,
delta: current - entry.value,
ratio: Number(ratio.toFixed(2)),
recommendedCeiling: Math.ceil(current * 1.15),
rationaleRequired: true,
});
}
}
return warnings;
}
2 changes: 2 additions & 0 deletions packages/adf/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ export { formatAdf } from './formatter';
export { applyPatches } from './patcher';
export { parseManifest, resolveModules, bundleModules } from './bundler';
export { validateConstraints, computeWeightSummary } from './validator';
export { evaluateEvidence } from './evidence';
export type { EvidenceReport, StaleBaselineWarning } from './evidence';
export { parseMarkdownSections } from './markdown-parser';
export type { MarkdownSection, MarkdownElement, RuleStrength } from './markdown-parser';
export { classifyElement, isDuplicateItem, buildMigrationPlan } from './content-classifier';
Expand Down
Loading