Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
322 changes: 182 additions & 140 deletions .github/workflows/benchmark.yml

Large diffs are not rendered by default.

14 changes: 6 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -423,14 +423,12 @@ Self-measured on every release via CI ([build benchmarks](generated/BUILD-BENCHM

| Metric | Latest |
|---|---|
| Build speed (native) | **1.9 ms/file** |
| Build speed (WASM) | **9.3 ms/file** |
| Query time | **3ms** |
| No-op rebuild (native) | **4ms** |
| 1-file rebuild (native) | **89ms** |
| Query: fn-deps | **2.1ms** |
| Query: path | **1.2ms** |
| ~50,000 files (est.) | **~95.0s build** |
| Build speed | **5.1 ms/file** |
| Query time | **2ms** |
| No-op rebuild | **5ms** |
| 1-file rebuild | **192ms** |
| Query: fn-deps | **0.5ms** |
| ~50,000 files (est.) | **~255.0s build** |

Metrics are normalized per file for cross-version comparability. Times above are for a full initial build — incremental rebuilds only re-parse changed files.

Expand Down
2 changes: 1 addition & 1 deletion crates/codegraph-core/src/complexity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -999,7 +999,7 @@ pub fn compute_all_metrics(
cognitive,
cyclomatic,
max_nesting,
halstead: Some(halstead.unwrap_or(HalsteadMetrics {
halstead: halstead.or(Some(HalsteadMetrics {
n1: 0, n2: 0, big_n1: 0, big_n2: 0,
vocabulary: 0, length: 0,
volume: 0.0, difficulty: 0.0, effort: 0.0, bugs: 0.0,
Expand Down
127 changes: 32 additions & 95 deletions generated/BUILD-BENCHMARKS.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ Metrics are normalized per file for cross-version comparability.

| Version | Engine | Date | Files | Build (ms/file) | Query (ms) | Nodes/file | Edges/file | DB (bytes/file) |
|---------|--------|------|------:|----------------:|-----------:|-----------:|-----------:|----------------:|
| 2.4.0 | native | 2026-02-27 | 122 | 1.9 ~ | 2.5 ↑67% | 6.4 ↑10% | 10.9 ↑20% | 5238 ↑36% |
| 2.4.0 | wasm | 2026-02-27 | 122 | 9.3 ↑41% | 3.4 ↑62% | 6.4 ↑10% | 10.9 ↑20% | 5506 ↑43% |
| 2.4.0 | wasm | 2026-02-28 | 123 | 5.1 ↓23% | 2.2 ↑5% | 6.5 ↑12% | 10.7 ↑18% | 4695 ↑22% |
| 2.3.0 | native | 2026-02-24 | 99 | 1.9 ~ | 1.5 ↑7% | 5.8 ↑7% | 9.1 ~ | 3848 ~ |
| 2.3.0 | wasm | 2026-02-24 | 99 | 6.6 ~ | 2.1 ↑11% | 5.8 ~ | 9.1 ↑3% | 3848 ~ |
| 2.1.0 | native | 2026-02-23 | 92 | 1.9 ↓24% | 1.4 ↑17% | 5.4 ↑6% | 9.1 ↓47% | 3829 ↓14% |
Expand All @@ -16,64 +15,39 @@ Metrics are normalized per file for cross-version comparability.

### Raw totals (latest)

#### Native (Rust)

| Metric | Value |
|--------|-------|
| Build time | 229ms |
| Query time | 3ms |
| Nodes | 778 |
| Edges | 1,333 |
| DB size | 624 KB |
| Files | 122 |

#### WASM

| Metric | Value |
|--------|-------|
| Build time | 1.1s |
| Query time | 3ms |
| Nodes | 778 |
| Edges | 1,333 |
| DB size | 656 KB |
| Files | 122 |

### Build Phase Breakdown (latest)

| Phase | Native | WASM |
|-------|-------:|-----:|
| Parse | 116 ms | 639.2 ms |
| Insert nodes | 14.1 ms | 15.9 ms |
| Resolve imports | 10.2 ms | 13.4 ms |
| Build edges | 61.9 ms | 59.1 ms |
| Structure | 4.1 ms | 7.3 ms |
| Roles | 4.7 ms | 5.2 ms |
| Complexity | 4.6 ms | 367.8 ms |
| Build time | 630ms |
| Query time | 2ms |
| Nodes | 801 |
| Edges | 1,320 |
| DB size | 564 KB |
| Files | 123 |

### Estimated performance at 50,000 files

Extrapolated linearly from per-file metrics above.

| Metric | Native (Rust) | WASM |
|--------|---:|---:|
| Build time | 95.0s | 465.0s |
| DB size | 249.8 MB | 262.5 MB |
| Nodes | 320,000 | 320,000 |
| Edges | 545,000 | 545,000 |
| Build time | n/a | 255.0s |
| DB size | n/a | 223.9 MB |
| Nodes | n/a | 325,000 |
| Edges | n/a | 535,000 |

### Incremental Rebuilds

| Version | Engine | No-op (ms) | 1-file (ms) |
|---------|--------|----------:|-----------:|
| 2.4.0 | native | 4 | 89 |
| 2.4.0 | wasm | 4 | 362 |
| 2.4.0 | wasm | 5 | 192 |

### Query Latency

| Version | Engine | fn-deps (ms) | fn-impact (ms) | path (ms) | roles (ms) |
|---------|--------|------------:|--------------:|----------:|----------:|
| 2.4.0 | native | 2.1 | 1.6 | 1.2 | 1.1 |
| 2.4.0 | wasm | 2.1 | 1.6 | 1.2 | 1.1 |
| 2.4.0 | wasm | 0.5 | 0.5 | null | 0.9 |

<!-- NOTES_START -->
### Notes
Expand All @@ -99,68 +73,31 @@ extractor is needed to recover the regression.
[
{
"version": "2.4.0",
"date": "2026-02-27",
"files": 122,
"date": "2026-02-28",
"files": 123,
"wasm": {
"buildTimeMs": 1138,
"queryTimeMs": 3.4,
"nodes": 778,
"edges": 1333,
"dbSizeBytes": 671744,
"buildTimeMs": 630,
"queryTimeMs": 2.2,
"nodes": 801,
"edges": 1320,
"dbSizeBytes": 577536,
"perFile": {
"buildTimeMs": 9.3,
"nodes": 6.4,
"edges": 10.9,
"dbSizeBytes": 5506
"buildTimeMs": 5.1,
"nodes": 6.5,
"edges": 10.7,
"dbSizeBytes": 4695
},
"noopRebuildMs": 4,
"oneFileRebuildMs": 362,
"noopRebuildMs": 5,
"oneFileRebuildMs": 192,
"queries": {
"fnDepsMs": 2.1,
"fnImpactMs": 1.6,
"pathMs": 1.2,
"rolesMs": 1.1
"fnDepsMs": 0.5,
"fnImpactMs": 0.5,
"pathMs": null,
"rolesMs": 0.9
},
"phases": {
"parseMs": 639.2,
"insertMs": 15.9,
"resolveMs": 13.4,
"edgesMs": 59.1,
"structureMs": 7.3,
"rolesMs": 5.2,
"complexityMs": 367.8
}
"phases": null
},
"native": {
"buildTimeMs": 229,
"queryTimeMs": 2.5,
"nodes": 778,
"edges": 1333,
"dbSizeBytes": 638976,
"perFile": {
"buildTimeMs": 1.9,
"nodes": 6.4,
"edges": 10.9,
"dbSizeBytes": 5238
},
"noopRebuildMs": 4,
"oneFileRebuildMs": 89,
"queries": {
"fnDepsMs": 2.1,
"fnImpactMs": 1.6,
"pathMs": 1.2,
"rolesMs": 1.1
},
"phases": {
"parseMs": 116,
"insertMs": 14.1,
"resolveMs": 10.2,
"edgesMs": 61.9,
"structureMs": 4.1,
"rolesMs": 4.7,
"complexityMs": 4.6
}
}
"native": null
},
{
"version": "2.3.0",
Expand Down
24 changes: 13 additions & 11 deletions scripts/benchmark.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,24 @@
import fs from 'node:fs';
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { fileURLToPath } from 'node:url';
import Database from 'better-sqlite3';
import { resolveBenchmarkSource, srcImport } from './lib/bench-config.js';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.resolve(__dirname, '..');

// Read version from package.json
const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
const { version, srcDir, cleanup } = await resolveBenchmarkSource();

const dbPath = path.join(root, '.codegraph', 'graph.db');

// Import programmatic API (use file:// URLs for Windows compatibility)
const { buildGraph } = await import(pathToFileURL(path.join(root, 'src', 'builder.js')).href);
const { buildGraph } = await import(srcImport(srcDir, 'builder.js'));
const { fnDepsData, fnImpactData, pathData, rolesData, statsData } = await import(
pathToFileURL(path.join(root, 'src', 'queries.js')).href
srcImport(srcDir, 'queries.js')
);
const { isNativeAvailable } = await import(
pathToFileURL(path.join(root, 'src', 'native.js')).href
srcImport(srcDir, 'native.js')
);

const INCREMENTAL_RUNS = 3;
Expand Down Expand Up @@ -133,10 +133,10 @@ async function benchmarkEngine(engine) {
}

const queries = {
fnDepsMs: benchQuery(fnDepsData, targets.hub, dbPath, { depth: 3, noTests: true }),
fnImpactMs: benchQuery(fnImpactData, targets.hub, dbPath, { depth: 3, noTests: true }),
pathMs: benchQuery(pathData, targets.hub, targets.leaf, dbPath, { noTests: true }),
rolesMs: benchQuery(rolesData, dbPath, { noTests: true }),
fnDepsMs: fnDepsData ? benchQuery(fnDepsData, targets.hub, dbPath, { depth: 3, noTests: true }) : null,
fnImpactMs: fnImpactData ? benchQuery(fnImpactData, targets.hub, dbPath, { depth: 3, noTests: true }) : null,
pathMs: pathData ? benchQuery(pathData, targets.hub, targets.leaf, dbPath, { noTests: true }) : null,
rolesMs: rolesData ? benchQuery(rolesData, dbPath, { noTests: true }) : null,
};

return {
Expand Down Expand Up @@ -173,7 +173,7 @@ if (isNativeAvailable()) {
console.log = origLog;

const result = {
version: pkg.version,
version,
date: new Date().toISOString().slice(0, 10),
files: wasm.files,
wasm: {
Expand Down Expand Up @@ -205,3 +205,5 @@ const result = {
};

console.log(JSON.stringify(result, null, 2));

cleanup();
9 changes: 6 additions & 3 deletions scripts/embedding-benchmark.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,16 @@ import path from 'node:path';
import { performance } from 'node:perf_hooks';
import { fileURLToPath } from 'node:url';
import Database from 'better-sqlite3';
import { resolveBenchmarkSource, srcImport } from './lib/bench-config.js';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.resolve(__dirname, '..');

const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
const { version, srcDir, cleanup } = await resolveBenchmarkSource();
const dbPath = path.join(root, '.codegraph', 'graph.db');

const { buildEmbeddings, MODELS, searchData, disposeModel } = await import(
new URL('../src/embedder.js', import.meta.url).href
srcImport(srcDir, 'embedder.js')
);

// Redirect console.log to stderr so only JSON goes to stdout
Expand Down Expand Up @@ -136,11 +137,13 @@ for (const key of modelKeys) {
console.log = origLog;

const output = {
version: pkg.version,
version,
date: new Date().toISOString().slice(0, 10),
strategy: 'structured',
symbols: symbols.length,
models: results,
};

console.log(JSON.stringify(output, null, 2));

cleanup();
18 changes: 11 additions & 7 deletions scripts/incremental-benchmark.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,22 @@
import fs from 'node:fs';
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { fileURLToPath } from 'node:url';
import { resolveBenchmarkSource, srcImport } from './lib/bench-config.js';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.resolve(__dirname, '..');

const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
const { version, srcDir, cleanup } = await resolveBenchmarkSource();
const dbPath = path.join(root, '.codegraph', 'graph.db');

const { buildGraph } = await import(pathToFileURL(path.join(root, 'src', 'builder.js')).href);
const { statsData } = await import(pathToFileURL(path.join(root, 'src', 'queries.js')).href);
const { buildGraph } = await import(srcImport(srcDir, 'builder.js'));
const { statsData } = await import(srcImport(srcDir, 'queries.js'));
const { resolveImportPath, resolveImportsBatch, resolveImportPathJS } = await import(
pathToFileURL(path.join(root, 'src', 'resolve.js')).href
srcImport(srcDir, 'resolve.js')
);
const { isNativeAvailable } = await import(
pathToFileURL(path.join(root, 'src', 'native.js')).href
srcImport(srcDir, 'native.js')
);

// Redirect console.log to stderr so only JSON goes to stdout
Expand Down Expand Up @@ -181,7 +183,7 @@ console.error(` native=${resolve.nativeBatchMs}ms js=${resolve.jsFallbackMs}ms`
console.log = origLog;

const result = {
version: pkg.version,
version,
date: new Date().toISOString().slice(0, 10),
files,
wasm: {
Expand All @@ -200,3 +202,5 @@ const result = {
};

console.log(JSON.stringify(result, null, 2));

cleanup();
Loading