From 5fe0a82e109d5939851510828f0b796a6686990c Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Tue, 3 Mar 2026 00:39:08 -0700 Subject: [PATCH 1/4] docs: update architecture audit and roadmap for v2.7.0 Add Phase 2.7 (Deep Analysis & Graph Enrichment) covering all features shipped since the last review: dataflow analysis, intraprocedural CFG, AST node storage, expanded node/edge types (13 kinds, 9 edge kinds), extractors refactoring, CLI consolidation, interactive viewer, exports command, and normalizeSymbol utility. Update Phase 3 priorities: add unified AST analysis framework as new #1 critical item (5,193 lines of parallel AST walking across complexity/ CFG/dataflow/ast with no shared infrastructure). Update all metrics across Phase 3 items (50 modules, 26K lines, 13 tables, 34 MCP tools). Mark qualified names as partially addressed (parent_id + contains edges). Subsume standalone complexity decomposition into AST framework. Update Phase 4 type definitions with new kind/edge type hierarchies. Mark Phase 9.1 visualization as partially complete (plot command). --- docs/roadmap/ROADMAP.md | 441 +++++++++++++------ generated/architecture.md | 872 +++++++++++++------------------------- 2 files changed, 610 insertions(+), 703 deletions(-) diff --git a/docs/roadmap/ROADMAP.md b/docs/roadmap/ROADMAP.md index 2e168e9e..163d1c9d 100644 --- a/docs/roadmap/ROADMAP.md +++ b/docs/roadmap/ROADMAP.md @@ -15,7 +15,8 @@ Codegraph is a strong local-first code graph CLI. This roadmap describes planned | [**1**](#phase-1--rust-core) | Rust Core | Rust parsing engine via napi-rs, parallel parsing, incremental tree-sitter, JS orchestration layer | **Complete** (v1.3.0) | | [**2**](#phase-2--foundation-hardening) | Foundation Hardening | Parser registry, complete MCP, test coverage, enhanced config, multi-repo MCP | **Complete** (v1.4.0) | | [**2.5**](#phase-25--analysis-expansion) | Analysis Expansion | Complexity metrics, community detection, flow tracing, co-change, manifesto, boundary rules, check, triage, audit, batch, hybrid search | **Complete** (v2.6.0) | -| [**3**](#phase-3--architectural-refactoring) | Architectural Refactoring | Command/query separation, repository pattern, queries.js decomposition, composable MCP, CLI commands, domain errors, curated API, unified graph model, dead symbol cleanup, community drift reduction, break function cycles | Planned | +| [**2.7**](#phase-27--deep-analysis--graph-enrichment) | Deep Analysis & Graph Enrichment | Dataflow analysis, intraprocedural CFG, AST node storage, expanded node/edge types, extractors refactoring, CLI consolidation, interactive viewer, exports command, normalizeSymbol | **Complete** (v2.7.0) | +| [**3**](#phase-3--architectural-refactoring) | Architectural Refactoring | Unified AST analysis framework, command/query separation, repository pattern, queries.js decomposition, composable MCP, CLI commands, domain errors, curated API, unified graph model | Planned | | [**4**](#phase-4--typescript-migration) | TypeScript Migration | Project setup, core type definitions, leaf -> core -> orchestration module migration, test migration | Planned | | [**5**](#phase-5--intelligent-embeddings) | Intelligent Embeddings | LLM-generated descriptions, enhanced embeddings, build-time semantic metadata, module summaries | Planned | | [**6**](#phase-6--natural-language-queries) | Natural Language Queries | `ask` command, conversational sessions, LLM-narrated graph queries, onboarding tools | Planned | @@ -29,11 +30,12 @@ Codegraph is a strong local-first code graph CLI. This roadmap describes planned Phase 1 (Rust Core) |--> Phase 2 (Foundation Hardening) |--> Phase 2.5 (Analysis Expansion) - |--> Phase 3 (Architectural Refactoring) - |--> Phase 4 (TypeScript Migration) - |--> Phase 5 (Embeddings + Metadata) --> Phase 6 (NL Queries + Narration) - |--> Phase 7 (Languages) - |--> Phase 8 (GitHub/CI) <-- Phase 5 (risk_score, side_effects) + |--> Phase 2.7 (Deep Analysis & Graph Enrichment) + |--> Phase 3 (Architectural Refactoring) + |--> Phase 4 (TypeScript Migration) + |--> Phase 5 (Embeddings + Metadata) --> Phase 6 (NL Queries + Narration) + |--> Phase 7 (Languages) + |--> Phase 8 (GitHub/CI) <-- Phase 5 (risk_score, side_effects) Phases 1-6 --> Phase 9 (Visualization + Refactoring Analysis) ``` @@ -355,19 +357,237 @@ MCP grew from 12 -> 25 tools, covering all new analysis capabilities. --- +## Phase 2.7 -- Deep Analysis & Graph Enrichment ✅ + +> **Status:** Complete -- shipped across PRs #254-#285 + +**Goal:** Add deeper static analysis capabilities (dataflow, control flow graphs, AST querying), enrich the graph model with sub-declaration node types and structural edges, refactor extractors into per-language modules, consolidate the CLI surface area, and introduce interactive visualization. This phase emerged from competitive analysis against Joern and Narsil-MCP. + +### 2.7.1 -- Dataflow Analysis ✅ + +Define-use chain extraction tracking how data flows between functions. + +- ✅ Three edge types: `flows_to` (parameter flow), `returns` (call return assignment), `mutates` (parameter-derived mutations) +- ✅ Confidence scoring (1.0 param, 0.9 call return, 0.8 destructured) +- ✅ Scope-stack AST walk with function-level tracking +- ✅ Forward BFS impact analysis via return value consumers +- ✅ Path finding through dataflow edges +- ✅ Opt-in via `build --dataflow` (dynamic import, only loaded when flag passed) +- ✅ DB migration v10: `dataflow` table with source, target, kind, param_index, expression, confidence +- ✅ JS/TS/TSX only (MVP -- language-specific scope analysis) +- ✅ CLI: `codegraph dataflow `, `codegraph dataflow-path `, `codegraph dataflow-impact ` +- ✅ MCP tools: `dataflow`, `dataflow_path`, `dataflow_impact` + +**New file:** `src/dataflow.js` (1,187 lines) + +### 2.7.2 -- Expanded Node Types (Phase 1) ✅ + +Extend the graph model with sub-declaration node kinds. + +- ✅ 3 new node kinds: `parameter`, `property`, `constant` +- ✅ Tiered constants: `CORE_SYMBOL_KINDS` (10), `EXTENDED_SYMBOL_KINDS` (3), `EVERY_SYMBOL_KIND` (13) +- ✅ Backward-compatible alias: `ALL_SYMBOL_KINDS = CORE_SYMBOL_KINDS` +- ✅ `parent_id` column on `nodes` table (DB migration v11) linking children to parent definitions +- ✅ All 9 WASM extractors updated to emit `children` arrays +- ✅ CLI: `codegraph children ` +- ✅ MCP tool: `symbol_children` with extended kind enum + +**Affected files:** All extractors, `src/builder.js`, `src/queries.js`, `src/db.js` + +### 2.7.3 -- Expanded Edge Types (Phase 2) ✅ + +Structural edges for richer graph relationships. + +- ✅ 3 new edge kinds: `contains` (parent→child), `parameter_of` (param→function), `receiver` (method call receiver) +- ✅ Tiered constants: `CORE_EDGE_KINDS` (6), `STRUCTURAL_EDGE_KINDS` (3), `EVERY_EDGE_KIND` (9) +- ✅ Structural edges excluded from `moduleMapData()` coupling counts +- ✅ MCP tool enums updated to include new edge kinds + +**Affected files:** `src/builder.js`, `src/queries.js` + +### 2.7.4 -- Intraprocedural Control Flow Graph (CFG) ✅ + +Basic-block control flow graph construction from function ASTs. + +- ✅ `makeCfgRules(overrides)` factory with per-language defaults and validation +- ✅ `CFG_RULES` Map covering all 9 languages (JS/TS, Python, Go, Rust, Java, C#, PHP, Ruby) +- ✅ Handles: if/else, for/while/do-while, switch, try/catch/finally, break/continue (with labels), return/throw +- ✅ Opt-in via `build --cfg` (dynamic import) +- ✅ DB migration v12: `cfg_blocks` and `cfg_edges` tables +- ✅ DOT and Mermaid export: `cfgToDOT()`, `cfgToMermaid()` +- ✅ CLI: `codegraph cfg ` with `--format text|dot|mermaid` +- ✅ MCP tool: `cfg` + +**New file:** `src/cfg.js` (1,451 lines) + +### 2.7.5 -- Stored Queryable AST Nodes ✅ + +Persist and query selected AST node types for pattern-based codebase exploration. + +- ✅ 6 AST node kinds: `call`, `new`, `string`, `regex`, `throw`, `await` +- ✅ `AST_NODE_KINDS` constant +- ✅ Pattern matching via SQL GLOB with auto-wrapping for substring search +- ✅ Parent resolution via narrowest enclosing definition +- ✅ Always-on extraction during build (post-parse, before complexity to preserve `_tree`) +- ✅ DB migration v13: `ast_nodes` table with indexes on kind, name, file, parent +- ✅ CLI: `codegraph ast [pattern]` with `-k`, `-f`, `-T`, `-j` +- ✅ MCP tool: `ast_query` + +**New file:** `src/ast.js` (392 lines) + +### 2.7.6 -- Extractors Refactoring ✅ + +Split per-language extractors from monolithic `parser.js` into dedicated modules. + +- ✅ New `src/extractors/` directory with 11 files (2,299 lines total) +- ✅ One file per language: `javascript.js` (750), `csharp.js` (248), `php.js` (243), `java.js` (230), `rust.js` (225), `ruby.js` (188), `go.js` (172), `python.js` (150), `hcl.js` (73) +- ✅ Shared utilities in `helpers.js` (`nodeEndLine()`, `findChild()`) +- ✅ Barrel export via `index.js` +- ✅ Consistent return schema: `{ definitions, calls, imports, classes, exports }` +- ✅ All extractors support extended node kinds (parameter, property, constant) +- ✅ `parser.js` reduced to thin WASM fallback with `LANGUAGE_REGISTRY` (404 lines) + +**New directory:** `src/extractors/` + +### 2.7.7 -- normalizeSymbol Utility ✅ + +Stable JSON schema for symbol output across all query functions. + +- ✅ `normalizeSymbol(row, db, hashCache)` returns 7-field shape: `{ name, kind, file, line, endLine, role, fileHash }` +- ✅ File hash caching for efficient batch operations +- ✅ Adopted by dataflow, context, where, query, and other functions + +**Affected file:** `src/queries.js` + +### 2.7.8 -- Interactive Graph Viewer ✅ + +Self-contained HTML visualization with vis-network. + +- ✅ File-level and function-level graph modes +- ✅ Layout switching (hierarchical, force, radial), physics toggle, search +- ✅ Color by kind/role/community/complexity (configurable) +- ✅ Size by uniform/fan-in/complexity +- ✅ Clustering by community or directory +- ✅ Drill-down with seed strategies (all, top-fanin, entry) +- ✅ Detail panel with metrics, callers, callees on node click +- ✅ Risk overlays (dead-code, high-blast-radius, low-MI) +- ✅ Configuration via `.plotDotCfg` / `.plotDotCfg.json` with deep merge defaults +- ✅ CLI: `codegraph plot` with `--functions`, `--config`, `--color-by`, `--size-by`, `--cluster-by`, `--overlay` + +**New file:** `src/viewer.js` (948 lines) + +### 2.7.9 -- Exports Command ✅ + +Per-symbol consumer analysis for file exports. + +- ✅ `exportsData(file)` returns each exported symbol with its consumers (who calls it and from where) +- ✅ CLI: `codegraph exports ` +- ✅ MCP tool: `file_exports` +- ✅ Integrated into batch command system + +**Affected file:** `src/queries.js` + +### 2.7.10 -- Export Format Expansion ✅ + +Three new graph export formats for external tooling integration. + +- ✅ GraphML (XML format for graph tools like yEd, Gephi) +- ✅ GraphSON (Gremlin/TinkerPop server format) +- ✅ Neo4j CSV (bulk loader format for Neo4j import) + +**Affected file:** `src/export.js` (681 lines) + +### 2.7.11 -- CLI Consolidation ✅ + +First CLI surface area reduction -- 5 commands merged into existing ones. + +- ✅ `hotspots` → folded into `triage --level file|directory` +- ✅ `manifesto` → merged into `check` (no args = manifesto, `--rules` for both) +- ✅ `explain` → replaced by `audit --quick` +- ✅ `batch-query` → use `batch where` instead +- ✅ `query --path` → standalone `path ` command (deprecation notice on old syntax) +- ✅ MCP tools unchanged for backward compatibility + +**Affected file:** `src/cli.js` + +### 2.7.12 -- MCP Tool Expansion ✅ + +MCP grew from 25 -> 34 tools, covering all new analysis capabilities. + +| New tool | Wraps | +|----------|-------| +| ✅ `cfg` | `cfgData` | +| ✅ `ast_query` | `astQueryData` | +| ✅ `dataflow` | `dataflowData` | +| ✅ `dataflow_path` | `dataflowPathData` | +| ✅ `dataflow_impact` | `dataflowImpactData` | +| ✅ `file_exports` | `exportsData` | +| ✅ `symbol_children` | `childrenData` | + +Plus updated enums on existing tools (edge_kinds, symbol kinds). + +**Affected file:** `src/mcp.js` (grew from 1,212 -> 1,370 lines) + +### 2.7 Summary + +| Metric | Before (v2.6.0) | After (v2.7.0) | Delta | +|--------|-----------------|-----------------|-------| +| Source modules | 35 | 50 | +15 | +| Total source lines | 17,830 | 26,277 | +47% | +| DB tables | 9 | 13 | +4 | +| DB migrations | v9 | v13 | +4 | +| MCP tools | 25 | 34 | +9 | +| CLI commands | 45 | 47 | +2 (net: +7 added, -5 consolidated) | +| Node kinds | 10 | 13 | +3 | +| Edge kinds | 6 | 9 | +3 | +| Test files | 59 | 70 | +11 | + +--- + ## Phase 3 -- Architectural Refactoring **Goal:** Restructure the codebase for modularity, testability, and long-term maintainability. These are internal improvements -- no new user-facing features, but they make every subsequent phase easier to build and maintain. > Reference: [generated/architecture.md](../../generated/architecture.md) -- full analysis with code examples and rationale. -**Context:** Phase 2.5 added 18 modules and doubled the codebase without introducing shared abstractions. The original Phase 3 recommendations (designed for a 5K-line codebase) are now even more urgent at 17,830 lines. The priority ordering has been revised based on the actual growth patterns. +**Context:** Phases 2.5 and 2.7 added 38 modules and grew the codebase from 5K to 26,277 lines without introducing shared abstractions. The dual-function anti-pattern was replicated across 19 modules. Three independent AST analysis engines (complexity, CFG, dataflow) totaling 4,801 lines share the same fundamental pattern but no infrastructure. Raw SQL is scattered across 25+ modules touching 13 tables. The priority ordering has been revised based on actual growth patterns -- the new #1 priority is the unified AST analysis framework. + +### 3.1 -- Unified AST Analysis Framework ★ Critical (New) + +Unify the three independent AST analysis engines (complexity, CFG, dataflow) plus AST node storage into a shared visitor framework. These four modules total 5,193 lines and independently implement the same pattern: per-language rules map → AST walk → collect data → write to DB → query → format. + +| Module | Lines | Languages | Pattern | +|--------|-------|-----------|---------| +| `complexity.js` | 2,163 | 8 | Per-language rules → AST walk → collect metrics | +| `cfg.js` | 1,451 | 9 | Per-language rules → AST walk → build basic blocks | +| `dataflow.js` | 1,187 | 1 (JS/TS) | Scope stack → AST walk → collect flows | +| `ast.js` | 392 | 1 (JS/TS) | AST walk → extract stored nodes | + +The extractors refactoring (Phase 2.7.6) proved the pattern: split per-language rules into files, share the engine. Apply it to all four AST analysis passes. + +``` +src/ + ast-analysis/ + visitor.js # Shared AST visitor with hook points + engine.js # Single-pass or multi-pass orchestrator + metrics.js # Halstead, MI, LOC/SLOC (language-agnostic) + cfg-builder.js # Basic-block + edge construction + rules/ + complexity/{lang}.js # Cognitive/cyclomatic rules per language + cfg/{lang}.js # Basic-block rules per language + dataflow/{lang}.js # Define-use chain rules per language + ast-store/{lang}.js # Node extraction rules per language +``` + +A single AST walk with pluggable visitors eliminates 3 redundant tree traversals per function, shares language-specific node type mappings, and allows new analyses to plug in without creating another 1K+ line module. -### 3.1 -- Command/Query Separation ★ Critical +**Affected files:** `src/complexity.js`, `src/cfg.js`, `src/dataflow.js`, `src/ast.js` -> split into `src/ast-analysis/` -Eliminate the `*Data()` / `*()` dual-function pattern replicated across 15 modules. Every analysis module (queries, audit, batch, check, cochange, communities, complexity, flow, manifesto, owners, structure, triage, branch-compare) currently implements both data extraction AND CLI formatting. +### 3.2 -- Command/Query Separation ★ Critical -Introduce a shared `CommandRunner` that handles the open-DB -> validate -> execute -> format -> paginate -> output lifecycle. Each command only implements unique query + analysis logic. Formatting is always separate and pluggable (CLI text, JSON, NDJSON, Mermaid). +Eliminate the `*Data()` / `*()` dual-function pattern replicated across 19 modules. Every analysis module (queries, audit, batch, check, cochange, communities, complexity, cfg, dataflow, ast, flow, manifesto, owners, structure, triage, branch-compare, viewer) currently implements both data extraction AND CLI formatting. + +Introduce a shared `CommandRunner` that handles the open-DB -> validate -> execute -> format -> paginate -> output lifecycle. Each command only implements unique query + analysis logic. Formatting is always separate and pluggable (CLI text, JSON, NDJSON, Mermaid, DOT). ``` src/ @@ -376,58 +596,65 @@ src/ impact.js audit.js check.js + cfg.js + dataflow.js ... infrastructure/ command-runner.js # Shared lifecycle - result-formatter.js # Shared formatting: table, JSON, NDJSON, Mermaid + result-formatter.js # Shared formatting: table, JSON, NDJSON, Mermaid, DOT test-filter.js # Shared --no-tests / isTestFile logic ``` -**Affected files:** All 15 modules with dual-function pattern, `src/cli.js`, `src/mcp.js` +**Affected files:** All 19 modules with dual-function pattern, `src/cli.js`, `src/mcp.js` -### 3.2 -- Repository Pattern for Data Access ★ Critical +### 3.3 -- Repository Pattern for Data Access ★ Critical -Consolidate all SQL into a single `Repository` class. Currently SQL is scattered across 20+ modules that each independently open the DB and write raw SQL inline. +Consolidate all SQL into a single `Repository` class. Currently SQL is scattered across 25+ modules that each independently open the DB and write raw SQL inline across 13 tables. ``` src/ db/ connection.js # Open, WAL mode, pragma tuning - migrations.js # Schema versions (currently 9 migrations) - repository.js # ALL data access methods across all 9+ tables + migrations.js # Schema versions (currently 13 migrations) + repository.js # ALL data access methods across all 13 tables query-builder.js # Lightweight SQL builder for common filtered queries ``` -Add a query builder for the common pattern "find nodes WHERE kind IN (...) AND file NOT LIKE '%test%' ORDER BY ... LIMIT ? OFFSET ?". Not an ORM -- a thin SQL builder that eliminates string construction across 20 modules. +Add a query builder for the common pattern "find nodes WHERE kind IN (...) AND file NOT LIKE '%test%' ORDER BY ... LIMIT ? OFFSET ?". Not an ORM -- a thin SQL builder that eliminates string construction across 25 modules. **Affected files:** `src/db.js` -> split into `src/db/`, SQL extracted from all modules -### 3.3 -- Decompose queries.js (3,110 Lines) +### 3.4 -- Decompose queries.js (3,395 Lines) Split into pure analysis modules that return data and share no formatting concerns. ``` src/ analysis/ - symbol-lookup.js # queryNameData, whereData, listFunctionsData + symbol-lookup.js # queryNameData, whereData, listFunctionsData, childrenData impact.js # impactAnalysisData, fnImpactData, diffImpactData dependencies.js # fileDepsData, fnDepsData, pathData module-map.js # moduleMapData, statsData context.js # contextData, explainData + exports.js # exportsData roles.js # rolesData shared/ - constants.js # SYMBOL_KINDS, ALL_SYMBOL_KINDS, VALID_ROLES - filters.js # isTestFile, normalizeSymbol, kindIcon + constants.js # CORE_SYMBOL_KINDS, EXTENDED_SYMBOL_KINDS, EVERY_SYMBOL_KIND, + # CORE_EDGE_KINDS, STRUCTURAL_EDGE_KINDS, EVERY_EDGE_KIND, + # VALID_ROLES, FALSE_POSITIVE_NAMES, AST_NODE_KINDS + normalize.js # normalizeSymbol, isTestFile, kindIcon generators.js # iterListFunctions, iterRoles, iterWhere ``` +> **Note:** Phase 2.7 introduced tiered constants (`CORE_`/`EXTENDED_`/`EVERY_`) and `normalizeSymbol()` — the right abstractions, just in the wrong file. Moving them to `shared/` is the first step. + **Affected files:** `src/queries.js` -> split into `src/analysis/` + `src/shared/` -### 3.4 -- Composable MCP Tool Registry +### 3.5 -- Composable MCP Tool Registry -Replace the monolithic 1,212-line `mcp.js` (25 tools in one switch dispatch) with self-contained tool modules. +Replace the monolithic 1,370-line `mcp.js` (34 tools in one switch dispatch) with self-contained tool modules. ``` src/ @@ -436,7 +663,7 @@ src/ tool-registry.js # Auto-discovery + dynamic registration middleware.js # Pagination, error handling, repo resolution tools/ - query-function.js # { schema, handler } -- one per tool (25 files) + query-function.js # { schema, handler } -- one per tool (34 files) ... ``` @@ -444,9 +671,11 @@ Adding a new MCP tool = adding a file. No other files change. **Affected files:** `src/mcp.js` -> split into `src/mcp/` -### 3.5 -- CLI Command Objects +### 3.6 -- CLI Command Objects -Move from 1,285 lines of inline Commander chains to self-contained command modules. +Move from 1,557 lines of inline Commander chains to self-contained command modules. + +> **Note:** Phase 2.7.11 consolidated 5 commands — the first CLI surface area reduction. This item continues that direction by making each of the 47 remaining commands independently testable. ``` src/ @@ -455,7 +684,7 @@ src/ shared/ output.js # --json, --ndjson, table, plain text options.js # Shared options (--no-tests, --json, --db, etc.) - commands/ # 45 files, one per command + commands/ # 47 files, one per command build.js # { name, description, options, validate, execute } ... ``` @@ -464,9 +693,9 @@ Each command is independently testable by calling `execute()` directly. **Affected files:** `src/cli.js` -> split into `src/cli/` -### 3.6 -- Curated Public API Surface +### 3.7 -- Curated Public API Surface -Reduce `index.js` from 120+ exports to ~30 curated exports. Use `package.json` `exports` field to enforce module boundaries. +Reduce `index.js` from 140+ exports to ~35 curated exports. Use `package.json` `exports` field to enforce module boundaries. ```json { "exports": { ".": "./src/index.js", "./cli": "./src/cli.js" } } @@ -476,9 +705,13 @@ Export only `*Data()` functions (the command execute functions). Never export CL **Affected files:** `src/index.js`, `package.json` -### 3.7 -- Domain Error Hierarchy +### ~~3.8~~ -- ~~Decompose complexity.js~~ Subsumed by 3.1 + +> The standalone complexity decomposition from the previous revision is now part of the unified AST analysis framework (3.1). The `complexity.js` per-language rules become `ast-analysis/rules/complexity/{lang}.js` alongside CFG and dataflow rules. -Replace ad-hoc error handling (mix of thrown `Error`, returned `null`, `logger.warn()`, `process.exit(1)`) across 35 modules with structured domain errors. +### 3.8 -- Domain Error Hierarchy + +Replace ad-hoc error handling (mix of thrown `Error`, returned `null`, `logger.warn()`, `process.exit(1)`) across 50 modules with structured domain errors. ```js class CodegraphError extends Error { constructor(message, { code, file, cause }) { ... } } @@ -495,35 +728,13 @@ The CLI catches domain errors and formats for humans. MCP returns structured err **New file:** `src/errors.js` -### 3.8 -- Decompose complexity.js (2,163 Lines) - -Split the largest source file into a rules/engine architecture mirroring the parser plugin concept. - -``` -src/ - complexity/ - index.js # Public API: computeComplexity, complexityData - metrics.js # Halstead, MI, LOC/SLOC (language-agnostic) - engine.js # Walk AST + apply rules -> raw values - rules/ - javascript.js # JS/TS/TSX rules - python.js - go.js - rust.js - java.js - csharp.js - php.js - ruby.js -``` - -**Affected files:** `src/complexity.js` -> split into `src/complexity/` - ### 3.9 -- Builder Pipeline Architecture -Refactor `buildGraph()` (1,173 lines) from a mega-function into explicit, independently testable pipeline stages. +Refactor `buildGraph()` (1,355 lines) from a mega-function into explicit, independently testable pipeline stages. Phase 2.7 added 4 opt-in stages, bringing the total to 11 core + 4 optional. ```js const pipeline = [ + // Core (always) collectFiles, // (rootDir, config) => filePaths[] detectChanges, // (filePaths, db) => { changed, removed, isFullBuild } parseFiles, // (filePaths, engineOpts) => Map @@ -533,10 +744,15 @@ const pipeline = [ buildClassEdges, // (symbolMap, nodeIndex) => classEdges[] resolveBarrels, // (edges, symbolMap) => resolvedEdges[] insertEdges, // (allEdges, db) => stats + extractASTNodes, // (fileSymbols, db) => astStats (always, post-parse) buildStructure, // (db, fileSymbols, rootDir) => structureStats classifyRoles, // (db) => roleStats - computeComplexity, // (db, rootDir, engine) => complexityStats emitChangeJournal, // (rootDir, changes) => void + + // Opt-in (dynamic imports) + computeComplexity, // --complexity: (db, rootDir, engine) => complexityStats + buildDataflowEdges, // --dataflow: (db, fileSymbols, rootDir) => dataflowStats + buildCFGData, // --cfg: (db, fileSymbols, rootDir) => cfgStats ] ``` @@ -572,7 +788,7 @@ The pluggable store interface enables future O(log n) ANN search (e.g., `hnswlib ### 3.11 -- Unified Graph Model -Unify the three parallel graph representations (structure.js, cochange.js, communities.js) into a shared in-memory graph model. +Unify the four parallel graph representations (structure.js, cochange.js, communities.js, viewer.js) into a shared in-memory graph model. ``` src/ @@ -596,11 +812,13 @@ src/ Algorithms become composable -- run community detection on the dependency graph, the temporal graph, or a merged graph. -**Affected files:** `src/structure.js`, `src/cochange.js`, `src/communities.js`, `src/cycles.js`, `src/triage.js` +**Affected files:** `src/structure.js`, `src/cochange.js`, `src/communities.js`, `src/cycles.js`, `src/triage.js`, `src/viewer.js` -### 3.12 -- Qualified Names & Hierarchical Scoping +### 3.12 -- Qualified Names & Hierarchical Scoping (Partially Addressed) -Enrich the node model with scope information to reduce ambiguity. +> **Phase 2.7 progress:** `parent_id` column, `contains` edges, `parameter_of` edges, and `childrenData()` query now model one-level parent-child relationships. This addresses ~80% of the use case. + +Remaining work -- enrich the node model with deeper scope information: ```sql ALTER TABLE nodes ADD COLUMN qualified_name TEXT; -- 'DateHelper.format' @@ -608,77 +826,22 @@ ALTER TABLE nodes ADD COLUMN scope TEXT; -- 'DateHelper' ALTER TABLE nodes ADD COLUMN visibility TEXT; -- 'public' | 'private' | 'protected' ``` -Enables queries like "all methods of class X" without traversing edges. Reduces reliance on heuristic confidence scoring. +Enables queries like "all methods of class X" without traversing edges. The `parent_id` FK only goes one level -- deeply nested scopes (namespace > class > method > closure) aren't fully represented. `qualified_name` would allow direct lookup. -**Affected files:** `src/db.js`, `src/parser.js` (extractors), `src/queries.js`, `src/builder.js` +**Affected files:** `src/db.js`, `src/extractors/`, `src/queries.js`, `src/builder.js` ### 3.13 -- Testing Pyramid with InMemoryRepository -The repository pattern (3.2) enables true unit testing: +The repository pattern (3.3) enables true unit testing: - Pure unit tests for graph algorithms (pass adjacency list, assert result) - Pure unit tests for risk/confidence scoring (pass parameters, assert score) - `InMemoryRepository` for query tests (no SQLite, instant setup) -- Existing 59 test files continue as integration tests +- Existing 70 test files continue as integration tests **Current gap:** Many "unit" tests still hit SQLite because there's no repository abstraction. -### 3.14 -- Dead Symbol Cleanup - -**Current state:** Role classification reports 221 dead symbols -- 27% of all classified code. Root causes: the dual-function pattern (every `*Data()` has an uncalled `*()` counterpart), 120+ speculative exports in `index.js`, and rapid feature addition without pruning. - -**Deliverables:** - -1. **Audit pass:** Categorize all dead symbols as truly dead (remove), entry points (annotate), or public API (keep or drop from `index.js`) -2. **Manifesto rule:** Add `max-dead-ratio` with `warn: 0.15`, `fail: 0.25` to prevent regression -3. **CI gate:** `codegraph check` fails if dead ratio exceeds threshold - -**Note:** Sections 3.1 (command/query separation) and 3.6 (curated API surface) eliminate the two biggest dead-code factories. This item captures the explicit cleanup and prevention gate. - -**Target:** Reduce dead symbol ratio from 27% to under 10%. - -**Affected files:** `src/manifesto.js`, `src/check.js`, dead code across all modules - -### 3.15 -- Community Drift Reduction - -**Current state:** Louvain community detection reports 40% drift -- files belong to a different logical community than their directory placement suggests. The flat `src/` layout with 35 modules gives no structural signal about which modules are coupled. - -**Deliverables:** - -1. **Directory restructuring:** Align file organization to detected communities (this happens naturally through 3.1-3.11): - ``` - src/ - analysis/ # Community: query/impact/context/explain/roles - commands/ # Community: CLI-specific formatting - health/ # Community: audit/triage/manifesto/check/complexity - graph/ # Community: structure/communities/cochange/cycles - infrastructure/ # Community: db/pagination/config/logger - ``` -2. **Track drift as a metric:** Add modularity score and drift percentage to `codegraph stats` output -3. **Manifesto rule:** Add `max-community-drift` with `warn: 0.30`, `fail: 0.45` - -**Target:** Reduce drift from 40% to under 20%. - -**Affected files:** `src/communities.js`, `src/manifesto.js`, `src/queries.js` (stats), directory structure - -### 3.16 -- Break Function-Level Cycles - -**Current state:** 9 function-level circular dependencies. File-level imports are acyclic, but function call graphs contain mutual recursion and indirect loops. These make impact analysis unreliable and complicate module decomposition. - -**Deliverables:** - -1. **Classify each cycle:** - - **Intentional recursion** (tree walkers, AST visitors) -- document and exempt from CI gate - - **Accidental coupling** (A→B→C→A) -- refactor by extracting shared logic or inverting dependencies - - **Layering violations** (query→builder→query) -- break with parameter passing or interface boundaries -2. **Break accidental cycles** through extraction, dependency inversion, or callback patterns -3. **CI gate:** Add `no-new-cycles` predicate to `codegraph check` at function scope - -**Target:** 0 accidental cycles (intentional recursion documented and exempted). - -**Affected files:** `src/check.js`, functions involved in the 9 cycles - -### 3.17 -- Remaining Items (Lower Priority) +### 3.14 -- Remaining Items (Lower Priority) These items from the original Phase 3 are still valid but less urgent: @@ -686,7 +849,7 @@ These items from the original Phase 3 are still valid but less urgent: - **Unified engine interface (Strategy):** Replace scattered `engine.name === 'native'` branching. Less critical now that native is the primary path. - **Subgraph export filtering:** `codegraph export --focus src/builder.js --depth 2` for usable visualizations. - **Transitive import-aware confidence:** Walk import graph before falling back to proximity heuristics. -- **Query result caching:** LRU/TTL cache between analysis layer and repository. More valuable now with 25 MCP tools. +- **Query result caching:** LRU/TTL cache between analysis layer and repository. More valuable now with 34 MCP tools. - **Configuration profiles:** `--profile backend` for monorepos with multiple services. - **Pagination standardization:** SQL-level LIMIT/OFFSET in repository + command runner shaping. @@ -715,14 +878,20 @@ Define TypeScript interfaces for all abstractions introduced in Phase 3: ```ts // Types for the core domain model -interface SymbolNode { id: number; name: string; qualifiedName?: string; kind: SymbolKind; file: string; line: number; endLine: number; } +interface SymbolNode { id: number; name: string; qualifiedName?: string; kind: SymbolKind; file: string; line: number; endLine: number; parentId?: number; } interface Edge { source: number; target: number; kind: EdgeKind; confidence: number; } -type SymbolKind = 'function' | 'method' | 'class' | 'interface' | 'type' | 'struct' | 'enum' | 'trait' | 'record' | 'module' -type EdgeKind = 'call' | 'import' | 'extends' | 'implements' +type CoreSymbolKind = 'function' | 'method' | 'class' | 'interface' | 'type' | 'struct' | 'enum' | 'trait' | 'record' | 'module' +type ExtendedSymbolKind = 'parameter' | 'property' | 'constant' +type SymbolKind = CoreSymbolKind | ExtendedSymbolKind +type CoreEdgeKind = 'imports' | 'imports-type' | 'reexports' | 'calls' | 'extends' | 'implements' +type StructuralEdgeKind = 'contains' | 'parameter_of' | 'receiver' +type EdgeKind = CoreEdgeKind | StructuralEdgeKind +type ASTNodeKind = 'call' | 'new' | 'string' | 'regex' | 'throw' | 'await' // Interfaces for Phase 3 abstractions interface Repository { insertNode(node: SymbolNode): void; findNodesByName(name: string, opts?: QueryOpts): SymbolNode[]; } interface Engine { parseFile(path: string, source: string): ParseResult; resolveImports(batch: ImportBatch): ResolvedImport[]; } +interface ASTVisitor { name: string; visit(node: TreeSitterNode, context: VisitorContext): void; } // Phase 3.1 interface Extractor { language: string; handlers: Record; } interface Command { name: string; options: OptionDef[]; validate(args: unknown, opts: unknown): void; execute(args: unknown, opts: unknown): Promise; } ``` @@ -1070,25 +1239,24 @@ Add SARIF output format for cycle detection. SARIF integrates with GitHub Code S ## Phase 9 -- Interactive Visualization & Advanced Features -### 9.1 -- Interactive Web Visualization +### 9.1 -- Interactive Web Visualization (Partially Complete) + +> **Phase 2.7 progress:** `codegraph plot` (Phase 2.7.8) ships a self-contained HTML viewer with vis-network. It supports layout switching, color/size/cluster overlays, drill-down, community detection, and a detail panel. The remaining work is the server-based experience below. ```bash codegraph viz ``` -Opens a local web UI at `localhost:3000` with: +Opens a local web UI at `localhost:3000` extending the static HTML viewer with: -- Force-directed graph layout (D3.js, inline -- no external dependencies) -- Zoom, pan, click-to-expand -- Node coloring by type (file=blue, function=green, class=purple) -- Edge styling by type (imports=solid, calls=dashed, extends=bold) -- Search bar for finding nodes by name +- Server-side filtering for large graphs (the current `plot` command embeds all data as JSON, scaling poorly past ~1K nodes) +- Lazy edge loading and progressive disclosure +- Code preview on hover (reads from source files via local server) - Filter panel: toggle node kinds, confidence thresholds, test files -- Code preview on hover (reads from source files) -- **Role-based coloring:** entry=orange, core=blue, utility=green, adapter=yellow, dead=gray (from structure.js roles) -- **Community overlay:** color by Louvain community assignment +- Edge styling by type (imports=solid, calls=dashed, extends=bold) +- Persistent view state (zoom, pan, expanded nodes saved across sessions) -**Data source:** Export JSON from DB, serve via lightweight HTTP server. +**Data source:** Serve from DB via lightweight HTTP server, lazy-load on interaction. **New file:** `src/visualizer.js` @@ -1101,7 +1269,7 @@ codegraph dead --exclude-exports --exclude-tests Find functions/methods/classes with zero incoming edges (never called). Filters for exports, test files, and entry points. -> **Note:** Phase 2.5 added role classification (`dead` role in structure.js) which provides the foundation. This extends it with a dedicated command and smarter filtering. +> **Note:** Phase 2.5 added role classification (`dead` role in structure.js) and Phase 2.7 added AST node storage (`ast_query` can find unreferenced exports). This extends those foundations with a dedicated command, smarter filtering, and cross-reference with `exports` command data. **Affected files:** `src/queries.js` @@ -1178,6 +1346,7 @@ Each phase includes targeted verification: | **1** | Benchmark native vs WASM parsing on a large repo, verify identical output from both engines | | **2** | `npm test`, manual MCP client test for all tools, config loading tests | | **2.5** | All 59 test files pass; integration tests for every new command; engine parity tests | +| **2.7** | All 70 test files pass; CFG + AST + dataflow integration tests; extractors produce identical output to pre-refactoring inline extractors | | **3** | All existing tests pass; each refactored module produces identical output to the pre-refactoring version; unit tests for pure analysis modules; InMemoryRepository tests | | **4** | `tsc --noEmit` passes with zero errors; all existing tests pass after migration; no runtime behavior changes | | **5** | Compare `codegraph search` quality before/after descriptions; verify `side_effects` and `risk_score` populated for LLM-enriched builds | diff --git a/generated/architecture.md b/generated/architecture.md index aea009a0..3011a997 100644 --- a/generated/architecture.md +++ b/generated/architecture.md @@ -2,718 +2,456 @@ > **Scope:** Unconstrained redesign proposals. No consideration for migration effort or backwards compatibility. What would the ideal architecture look like? > -> **Revision context:** The original audit (Feb 22, 2026) analyzed v1.4.0 with ~12 source modules totaling ~5K lines. Since then, the codebase grew to v2.6.0 with 35 source modules totaling 17,830 lines — a 3.5x expansion. 18 new modules were added, MCP tools went from 12 to 25, CLI commands from ~20 to 45, and `index.js` exports from ~40 to 120+. This revision re-evaluates every recommendation against the actual codebase as it stands today. +> **Revision context:** The original audit (Feb 22, 2026) analyzed v1.4.0 with ~12 source modules totaling ~5K lines. The first revision (Mar 2, 2026) covered v2.6.0 with 35 modules totaling 17,830 lines. Since then, a rapid expansion added 6 new modules (cfg, ast, dataflow, viewer, extractors refactor, CLI consolidation), 4 new DB tables, 3 new node kinds, 3 new edge kinds, and 9 new MCP tools — all in a single day. The codebase now stands at 50 source modules totaling 26,277 lines. This revision re-evaluates every recommendation against the actual codebase as it stands today. --- -## What Changed Since the Original Audit +## What Changed Since the Last Revision (Mar 2 → Mar 3, 2026) -Before diving into recommendations, here's what happened: +| Metric | Mar 2 (v2.6.0) | Mar 3 (post-PRs) | Delta | +|--------|----------------|-------------------|-------| +| Source modules | 35 | 50 (37 core + 11 extractors + 2 new) | +15 | +| Total source lines | 17,830 | 26,277 | +47% | +| `queries.js` | 3,110 lines | 3,395 lines | +285 | +| `mcp.js` | 1,212 lines | 1,370 lines | +158 | +| `cli.js` | 1,285 lines | 1,557 lines | +272 | +| `builder.js` | 1,173 lines | 1,355 lines | +182 | +| `cfg.js` | -- | 1,451 lines | New | +| `dataflow.js` | -- | 1,187 lines | New | +| `viewer.js` | -- | 948 lines | New | +| `ast.js` | -- | 392 lines | New | +| `db.js` | 317 lines | 392 lines | +75 | +| `export.js` | 681 lines | 681 lines | unchanged | +| DB tables | 9 | 13 | +4 | +| DB migrations | v9 | v13 | +4 | +| MCP tools | 25 | 34 | +9 | +| CLI commands | 45 | 47 | +2 (net: +7 added, -5 consolidated) | +| `index.js` exports | 120+ | 140+ (32 export lines) | +20 | +| Test files | 59 | 70 | +11 | +| Node kinds | 10 | 13 | +3 (parameter, property, constant) | +| Edge kinds | 6 | 9 | +3 (contains, parameter_of, receiver) | +| Extractor modules | 0 (inline in parser.js) | 11 files, 2,299 lines | New directory | -| Metric | Feb 2026 (v1.4.0) | Mar 2026 (v2.6.0) | Growth | -|--------|-------------------|-------------------|--------| -| Source modules | ~12 | 35 | 2.9x | -| Total source lines | ~5,000 | 17,830 | 3.5x | -| `queries.js` | 823 lines | 3,110 lines | 3.8x | -| `mcp.js` | 354 lines | 1,212 lines | 3.4x | -| `cli.js` | -- | 1,285 lines | -- | -| `builder.js` | 554 lines | 1,173 lines | 2.1x | -| `embedder.js` | 525 lines | 1,113 lines | 2.1x | -| `complexity.js` | -- | 2,163 lines | New | -| MCP tools | 12 | 25 | 2.1x | -| CLI commands | ~20 | 45 | 2.3x | -| `index.js` exports | ~40 | 120+ | 3x | -| Test files | ~15 | 59 | 3.9x | +**Key patterns observed in this burst:** -**Key pattern observed:** Every new feature (audit, batch, boundaries, check, cochange, communities, complexity, flow, manifesto, owners, structure, triage) was added as a standalone module following the same internal pattern: raw SQL + BFS/traversal logic + CLI formatting + JSON output + `*Data()` / `*()` dual functions. No shared abstractions were introduced. The original architectural debt wasn't addressed -- it was replicated 15 times. +1. **The dual-function anti-pattern was replicated 4 more times** (cfg.js, ast.js, dataflow.js, viewer.js) — each with its own `*Data()` / `*()` pair, DB opening, SQL, formatting. The pattern count went from 15 to 19 modules. ---- - -## 1. The Dual-Function Anti-Pattern Is Now the Dominant Architecture Problem +2. **CFG introduced a third analysis engine pattern** alongside complexity and dataflow: language-specific rule maps keyed by AST node type, applied during a tree walk. Three modules now independently implement "per-language AST rules + engine walker" with no shared framework. -**Original analysis (S3):** `queries.js` mixes data access, graph algorithms, and presentation. The `*Data()` / `*()` dual-function pattern was identified as a workaround for coupling. +3. **The extractors refactoring (PR #270) is the first genuine structural decomposition** — parser.js extractors split into `src/extractors/` with one file per language. This is the pattern the rest of the codebase should follow. -**What happened:** Every new module adopted the same pattern. There are now **15+ modules** each implementing both data extraction AND CLI formatting: +4. **Scope and parent hierarchy finally arrived** — `parent_id` column on `nodes`, `contains`/`parameter_of` edges, `children` query. This partially addresses the qualified names gap (item #11 in the previous revision). -``` -queries.js -> queryNameData() / queryName(), impactAnalysisData() / impactAnalysis(), ... -audit.js -> auditData() / audit() -batch.js -> batchData() / batch() -check.js -> checkData() / check() -cochange.js -> coChangeData() / coChange(), coChangeTopData() / coChangeTop() -communities.js -> communitiesData() / communities() -complexity.js -> complexityData() / complexity() -flow.js -> flowData() / flow() -manifesto.js -> manifestoData() / manifesto() -owners.js -> ownersData() / owners() -structure.js -> structureData() / structure(), hotspotsData() / hotspots() -triage.js -> triageData() / triage() -branch-compare -> branchCompareData() / branchCompare() -``` - -Each of these modules independently handles: DB opening, SQL execution, result shaping, pagination integration, CLI formatting, JSON output, and `--no-tests` filtering. The repetition is massive. - -**Ideal architecture -- Command + Query separation with shared infrastructure:** - -``` -src/ - commands/ # One file per command - query.js # { execute(args, ctx) -> data, format(data, opts) -> string } - impact.js - audit.js - check.js - ... - - infrastructure/ - command-runner.js # Shared lifecycle: open DB -> validate -> execute -> format -> paginate - result-formatter.js # Shared formatting: table, JSON, NDJSON, Mermaid - pagination.js # Shared pagination with consistent interface - test-filter.js # Shared --no-tests / isTestFile logic - - analysis/ # Pure algorithms -- no I/O, no formatting - bfs.js # Graph traversals (BFS, DFS, shortest path) - impact.js # Blast radius computation - confidence.js # Import resolution scoring - clustering.js # Community detection, coupling analysis - risk.js # Triage scoring, hotspot detection -``` - -The key insight: every command follows the same lifecycle -- `(args) -> open DB -> query -> analyze -> format -> output`. A shared `CommandRunner` handles the lifecycle. Each command only implements the unique query + analysis logic. Formatting is always separate and pluggable (CLI text, JSON, NDJSON, Mermaid). - -This eliminates the dual-function pattern entirely. `index.js` exports `auditData` (the command's execute function) -- the CLI formatter is internal to the CLI layer and never exported. +5. **CLI consolidation (PR #280) removed 5 commands** — the first time the project actively reduced surface area. `hotspots` merged into `triage`, `manifesto` into `check`, `explain` into `audit --quick`, `batch-query` into `batch where`, `query --path` into standalone `path`. --- -## 2. The Database Layer Needs a Repository -- Now More Than Ever - -**Original analysis (S2):** SQL scattered across `builder.js`, `queries.js`, `embedder.js`, `watcher.js`, `cycles.js`. +## 1. The Dual-Function Anti-Pattern — Now 19 Modules Deep -**What happened:** SQL is now scattered across **20+ modules**: all of the above plus `audit.js`, `check.js`, `cochange.js`, `communities.js`, `complexity.js`, `flow.js`, `manifesto.js`, `owners.js`, `structure.js`, `triage.js`, `snapshot.js`, `branch-compare.js`. Each module opens the DB independently with `openDb()`, creates its own prepared statements, and writes raw SQL inline. +**Previous state:** 15 modules with `*Data()` / `*()` pairs. -The schema grew to 9 tables: `nodes`, `edges`, `node_metrics`, `file_hashes`, `co_changes`, `co_change_meta`, `file_commit_counts`, `build_meta`, `function_complexity`. Plus embeddings and FTS5 tables in `embedder.js`. - -**Ideal architecture** (unchanged from original, but now higher priority): +**Current state:** 19 modules. Four new additions: ``` -src/ - db/ - connection.js # Open, WAL mode, pragma tuning, connection pooling - migrations.js # Schema versions (currently 9 migrations) - repository.js # ALL read/write operations across all 9+ tables - types.js # JSDoc type definitions for all entities +cfg.js -> cfgData() / cfg() +ast.js -> astQueryData() / astQuery() +dataflow.js -> dataflowData() / dataflow(), dataflowPathData(), dataflowImpactData() +viewer.js -> prepareGraphData() / generatePlotHTML() ``` -**New addition -- query builders for common patterns:** - -Many modules do the same filtered query: "find nodes WHERE kind IN (...) AND file NOT LIKE '%test%' AND name LIKE ? ORDER BY ... LIMIT ? OFFSET ?". A lightweight query builder eliminates this SQL duplication: +Plus queries.js grew two more pairs: `childrenData()` / `children()`, `exportsData()` / `fileExports()`. -```js -repo.nodes() - .where({ kind: ['function', 'method'], file: { notLike: '%test%' } }) - .matching(name) - .orderBy('name') - .paginate(opts) - .all() -``` +**Reinforced assessment:** Each new module independently handles DB opening, SQL execution, result shaping, pagination, CLI formatting, JSON output, and `--no-tests` filtering. The `cfg.js` module at 1,451 lines is the most extreme example — it contains CFG construction rules for 9 languages, a build phase, a query function, DOT/Mermaid formatters, and a CLI printer all in one file. -Not an ORM -- a thin SQL builder that generates the same prepared statements but eliminates string construction across 20 modules. +**The ideal architecture is unchanged** — Command + Query separation with shared `CommandRunner` lifecycle. But the urgency increased: at the current rate of ~4 new dual-function modules per development sprint, the pattern will reach 25+ modules before any refactoring can happen. --- -## 3. queries.js at 3,110 Lines Must Be Decomposed - -**Original analysis (S3):** 823 lines mixing data access, algorithms, and presentation. +## 2. The Database Layer — 13 Tables Across 25+ Modules -**Current state:** 3,110 lines -- nearly 4x growth. Contains 15+ data functions, 15+ display functions, constants (`SYMBOL_KINDS`, `ALL_SYMBOL_KINDS`, `VALID_ROLES`, `FALSE_POSITIVE_NAMES`), icon helpers (`kindIcon`), normalization (`normalizeSymbol`), test filtering (`isTestFile`), and generator functions (`iterListFunctions`, `iterRoles`, `iterWhere`). +**Previous state:** 9 tables, SQL scattered across 20+ modules. -This is now the second-largest file in the codebase (after `complexity.js` at 2,163 lines) and the most interconnected -- almost every other module imports from it. +**Current state:** 13 tables, SQL scattered across **25+ modules**. New tables: -**Ideal decomposition:** +| Table | Migration | Module | Purpose | +|-------|-----------|--------|---------| +| `dataflow` | v10 | `dataflow.js` | flows_to, returns, mutates edges with confidence | +| `nodes.parent_id` | v11 | `builder.js` | Parent-child node hierarchy | +| `cfg_blocks` | v12 | `cfg.js` | Basic blocks per function | +| `cfg_edges` | v12 | `cfg.js` | Control flow edges between blocks | +| `ast_nodes` | v13 | `ast.js` | Stored queryable AST nodes (call, new, string, regex, throw, await) | -``` -src/ - analysis/ - symbol-lookup.js # queryNameData, whereData, listFunctionsData - impact.js # impactAnalysisData, fnImpactData, diffImpactData - dependencies.js # fileDepsData, fnDepsData, pathData - module-map.js # moduleMapData, statsData - context.js # contextData, explainData - roles.js # rolesData (currently delegates to structure.js) - - shared/ - constants.js # SYMBOL_KINDS, ALL_SYMBOL_KINDS, VALID_ROLES, FALSE_POSITIVE_NAMES - filters.js # isTestFile, normalizeSymbol, kindIcon - generators.js # iterListFunctions, iterRoles, iterWhere -``` +Each new module follows the same pattern: import `openDb()`, write raw SQL with inline string construction, create its own prepared statements. `cfg.js` alone has ~20 SQL statements. -Each analysis module is purely data -- no CLI output, no JSON formatting, no `console.log`. The `*Data()` suffix disappears because there's no `*()` counterpart. These are just functions that return data. +**The repository pattern is now even more critical.** With 13 tables, the migration system in `db.js` is getting complex (392 lines, up from 317). The ideal decomposition into `db/connection.js`, `db/migrations.js`, `db/repository.js` is unchanged but higher urgency. --- -## 4. MCP at 1,212 Lines with 25 Tools Needs Composability +## 3. queries.js at 3,395 Lines — Still Growing -**Original analysis (S10):** 354 lines, 12 tools, monolithic switch dispatch. +**Previous state:** 3,110 lines. -**Current state:** 1,212 lines, 25 tools. The `buildToolList()` function dynamically builds tool definitions, and a large switch/dispatch handles all 25 tools. Adding a tool still requires editing the tool list, the dispatch block, and importing the handler -- three changes in one file. +**Current state:** 3,395 lines — gained 285 lines. New additions: +- `childrenData()` — query child symbols (parameters, properties, constants) +- `exportsData()` — per-symbol consumer analysis for file exports +- `CORE_SYMBOL_KINDS` (10) / `EXTENDED_SYMBOL_KINDS` (3) / `EVERY_SYMBOL_KIND` (13) — tiered kind constants +- `CORE_EDGE_KINDS` (6) / `STRUCTURAL_EDGE_KINDS` (3) / `EVERY_EDGE_KIND` (9) — tiered edge constants +- `normalizeSymbol()` — stable 7-field JSON shape for all queries -**Ideal architecture** (unchanged from original, now critical): +**Positive development:** The constant hierarchy (`CORE_` / `EXTENDED_` / `EVERY_`) is well-designed and provides clean backward compatibility (`ALL_SYMBOL_KINDS = CORE_SYMBOL_KINDS`). The `normalizeSymbol()` utility enforces consistent output. These are **the right abstractions** — they just need to live in dedicated files (`shared/constants.js`, `shared/normalize.js`) rather than accumulating in the megafile. -``` -src/ - mcp/ - server.js # MCP server setup, transport, connection lifecycle - tool-registry.js # Auto-discovery + dynamic registration - middleware.js # Pagination, error handling, repo resolution - tools/ - query-function.js # { schema, handler } - file-deps.js - impact-analysis.js - check.js - audit.js - complexity.js - co-changes.js - structure.js - ... (25 files, one per tool) -``` +**The decomposition plan from the previous revision still applies.** Add `shared/constants.js` for the kind/edge/role constants and `shared/normalize.js` for `normalizeSymbol` + `isTestFile` + `kindIcon`. -Each tool is self-contained: +--- -```js -export const schema = { - name: 'audit', - description: '...', - inputSchema: { ... } -} - -export async function handler(args, context) { - return auditData(args.target, context.resolveDb(args.repo), args) -} -``` +## 4. MCP at 1,370 Lines with 34 Tools -The registry auto-discovers tools from the directory. Shared middleware handles pagination (the `MCP_DEFAULTS` logic currently in `paginate.js`), error wrapping, and multi-repo resolution. Adding a tool = adding a file. +**Previous state:** 1,212 lines, 25 tools. ---- +**Current state:** 1,370 lines, 34 tools. Nine new tools added: -## 5. CLI at 1,285 Lines with 45 Commands Needs Command Objects +| Tool | Source module | +|------|-------------| +| `cfg` | cfg.js | +| `ast_query` | ast.js | +| `dataflow` | dataflow.js | +| `dataflow_path` | dataflow.js | +| `dataflow_impact` | dataflow.js | +| `file_exports` | queries.js | +| `symbol_children` | queries.js | +| `fn_impact` (extended kinds enum) | queries.js | +| Various updated enums | (edge_kinds, symbol kinds) | -**Original analysis (S12):** CLI was mentioned as a future concern. +**Positive development:** The MCP tools were **not** consolidated alongside the CLI (PR #280 removed 5 CLI commands but kept all MCP tools for backward compatibility). This is the right call for an MCP API — clients may depend on specific tool names. -**Current state:** 1,285 lines of inline Commander.js chains. 45 commands registered with `.command().description().option().action()` patterns. Each action handler directly calls module functions, handles `--json` output, and manages error display. +**The composable tool registry pattern is now more urgent.** At 34 tools in a single file, each addition requires coordinating the tool definition, the dispatch handler, and the import — three touch points. The one-file-per-tool registry pattern proposed in the previous revision would make each of the 34 tools independently maintainable. -**Ideal architecture:** +--- -``` -src/ - cli/ - index.js # Commander setup, auto-discover commands - shared/ - output.js # --json, --ndjson, table, plain text output - options.js # Shared options (--no-tests, --json, --db, --engine, --limit, --offset) - validation.js # Argument validation, path resolution - commands/ - build.js # { name, description, options, validate, execute } - query.js - impact.js - audit.js - check.js - ... (45 files) -``` +## 5. CLI at 1,557 Lines with 47 Commands — Consolidation Started -Each command: +**Previous state:** 1,285 lines, 45 commands. -```js -export default { - name: 'audit', - description: 'Combined explain + impact + health report', - arguments: [{ name: 'target', required: true }], - options: [ - { flags: '-T, --no-tests', description: 'Exclude test files' }, - { flags: '-j, --json', description: 'JSON output' }, - { flags: '--db ', description: 'Custom DB path' }, - ], - async execute(args, opts) { - const data = await auditData(args.target, opts.db, opts) - return data // CommandRunner handles formatting - }, -} -``` +**Current state:** 1,557 lines, 47 commands. Net change: +7 new commands (cfg, ast, dataflow, dataflow-path, dataflow-impact, children, path), -5 consolidated commands (hotspots, manifesto, explain, batch-query, query --path). -The CLI index auto-discovers commands. Shared options (`--no-tests`, `--json`, `--db`, `--engine`, `--limit`, `--offset`) are applied uniformly. The `CommandRunner` handles the open-DB -> execute -> format -> output lifecycle. +**Positive development:** PR #280 is the first CLI surface area reduction — 5 commands consolidated into existing ones. This is the right direction. `check` now subsumes `manifesto`, `triage` subsumes `hotspots`, `audit --quick` subsumes `explain`, `batch where` subsumes `batch-query`. + +**But the file still grew** because 7 new commands were added in parallel. The inline Commander.js pattern means each new command adds 20-40 lines of `.command().description().option().action()` boilerplate. The command object pattern from the previous revision would keep the entry point lean regardless of command count. --- -## 6. complexity.js at 2,163 Lines Is a Hidden Monolith +## 6. cfg.js at 1,451 Lines — A New Monolith + +**Not in previous revision** — this module didn't exist. -**Not in original analysis** -- this module didn't exist in Feb 2026. +**Current state:** 1,451 lines containing: +- `makeCfgRules(overrides)` — factory for language-specific CFG construction rules +- `CFG_RULES` Map — rules for all 9 supported languages (JS/TS, Python, Go, Rust, Java, C#, PHP, Ruby) +- `buildFunctionCFG(functionNode, langId)` — CFG construction from AST (basic blocks + control flow edges) +- `buildCFGData(db, fileSymbols, rootDir)` — build-phase integration (write cfg_blocks/cfg_edges to DB) +- `cfgData(name, customDbPath, opts)` — query function +- `cfgToDOT()` / `cfgToMermaid()` — graph export formatters +- `cfg(name, customDbPath, opts)` — CLI printer -**Current state:** 2,163 lines containing language-specific AST complexity rules for 8 languages (JS/TS, Python, Go, Rust, Java, C#, PHP, Ruby), plus Halstead metrics computation, maintainability index calculation, LOC/SLOC counting, and CLI formatting. It's the largest file in the codebase. +**Problem:** This is a miniature version of the `complexity.js` monolith. It has the same structure: per-language rules map + engine walker + DB integration + query + formatting. The two modules share the same fundamental pattern but implement it independently. -**Problem:** The file is structured as a giant map of language to rules, but the rules for each language are deeply nested objects with inline AST traversal logic. Adding a new language or modifying a rule requires working inside a 2K-line file. +**Connection to complexity.js:** `cfg.js` imports `findFunctionNode()` from `complexity.js` — confirming that these two AST-analysis modules have shared concerns but no shared framework. -**Ideal architecture:** +**Ideal architecture — unified AST analysis framework:** ``` src/ - complexity/ - index.js # Public API: computeComplexity, complexityData - metrics.js # Halstead, MI, LOC/SLOC computation (language-agnostic) - engine.js # Walk AST + apply rules -> raw metric values + ast-analysis/ + engine.js # Shared AST walk with visitor pattern rules/ - javascript.js # JS/TS/TSX complexity rules - python.js - go.js - rust.js - java.js - csharp.js - php.js - ruby.js + complexity/ # Cognitive/cyclomatic/Halstead rules per language + javascript.js + python.js + ... + cfg/ # Basic-block construction rules per language + javascript.js + python.js + ... + metrics.js # Halstead, MI computation (from complexity.js) + cfg-builder.js # Basic-block + edge construction (from cfg.js) ``` -Each rules file exports a declarative complexity rule set. The engine applies rules to AST nodes. Metrics computation is shared. This mirrors the parser plugin system concept -- same pattern, applied to complexity. +Both complexity and CFG analysis walk the same AST trees with language-specific rules. A shared visitor-pattern engine would eliminate the parallel rule-map implementations and allow future AST analyses (e.g., dead code detection, mutation analysis) to plug in without creating yet another 1K+ line module. --- -## 7. builder.js at 1,173 Lines -- Pipeline Architecture +## 7. dataflow.js at 1,187 Lines — JS/TS Only, Language Hardcoding -**Original analysis (S4):** 554 lines, mega-function that's hard to test in parts. +**Not in previous revision** — this module was just introduced (#254). -**Current state:** 1,173 lines -- doubled. Now includes change journal integration, structure building, role classification, incremental verification, and more complex edge building. The `buildGraph()` function is even more of a mega-function. +**Current state:** 1,187 lines implementing define-use chain extraction with three edge types: +- `flows_to` — parameter/variable flow between functions +- `returns` — call return value assignment tracking +- `mutates` — parameter-derived mutation detection -**Ideal architecture** (unchanged, reinforced): +**Design qualities:** +- Confidence scoring (1.0 param, 0.9 call return, 0.8 destructured) — good, but undocumented +- Transaction-based DB writes — correct pattern +- Lazy parser initialization — efficient -```js -const pipeline = [ - collectFiles, // (rootDir, config) => filePaths[] - detectChanges, // (filePaths, db) => { changed, removed, isFullBuild } - parseFiles, // (filePaths, engineOpts) => Map - insertNodes, // (symbolMap, db) => nodeIndex - resolveImports, // (symbolMap, rootDir, aliases) => importEdges[] - buildCallEdges, // (symbolMap, nodeIndex) => callEdges[] - buildClassEdges, // (symbolMap, nodeIndex) => classEdges[] - resolveBarrels, // (edges, symbolMap) => resolvedEdges[] - insertEdges, // (allEdges, db) => stats - buildStructure, // (db, fileSymbols, rootDir) => structureStats - classifyRoles, // (db) => roleStats - computeComplexity, // (db, rootDir, engine) => complexityStats - emitChangeJournal, // (rootDir, changes) => void -] -``` - -The pipeline grew -- four new stages since the original analysis. This reinforces the need: each stage is independently testable and the pipeline runner handles transactions, logging, progress, and statistics. +**Architectural concerns:** +1. **Language hardcoding** — Lines 517-524 and 573-580 hardcode `javascript`/`typescript`/`tsx` checks. Not extensible via registry. +2. **Scope stack mutation** during tree walk — fragile for malformed AST +3. **No cycle detection** in dataflow BFS paths — can revisit nodes +4. **Statement-level mutation detection** misses inline mutations +5. **Follows the same monolith pattern** — extraction + DB write + query + CLI format all in one file -**Watch mode** reuses the same stages triggered per-file, eliminating the `watcher.js` divergence. `change-journal.js` and `journal.js` integrate as pipeline hooks rather than separate code paths. +**Ideal:** Dataflow extraction should integrate with the AST analysis framework proposed above. The define-use chain walk is fundamentally the same visitor pattern as complexity and CFG — it just collects different data. --- -## 8. embedder.js at 1,113 Lines -- Now Includes Three Search Engines +## 8. Extractors Refactoring — The Right Pattern, Applied Once -**Original analysis (S5):** 525 lines, mini vector database bolted onto the graph DB. +**Previous state:** parser.js at 404 lines with inline extractors. -**Current state:** 1,113 lines. Now contains: -- 8 embedding model definitions with batch sizes and dimensions -- 2 embedding strategies (structured, source) -- Vector storage in SQLite blobs -- Cosine similarity search (O(n) linear scan) -- **FTS5 full-text index with BM25 scoring** (new) -- **Hybrid search with RRF fusion** (new) -- Model lifecycle management (lazy loading, caching) +**Current state:** `src/extractors/` directory with 11 files totaling 2,299 lines: -Hybrid search (originally planned as Phase 5.3) is already implemented -- but inside the monolith. +| File | Lines | Language | +|------|-------|----------| +| `javascript.js` | 750 | JS/TS/TSX | +| `csharp.js` | 248 | C# | +| `php.js` | 243 | PHP | +| `java.js` | 230 | Java | +| `rust.js` | 225 | Rust | +| `ruby.js` | 188 | Ruby | +| `go.js` | 172 | Go | +| `python.js` | 150 | Python | +| `hcl.js` | 73 | Terraform/HCL | +| `helpers.js` | 11 | Shared utilities | +| `index.js` | 9 | Barrel export | -**Ideal architecture** (updated): +**This is the correct decomposition pattern.** Each language has its own file. A shared helpers module provides `nodeEndLine()` and `findChild()`. The barrel export keeps the public API clean. All extractors return a consistent structure: `{ definitions, calls, imports, classes, exports }`. -``` -src/ - embeddings/ - index.js # Public API - models.js # Model definitions, batch sizes, loading - generator.js # Source -> text preparation -> batch embedding - stores/ - sqlite-blob.js # Current O(n) cosine similarity - fts5.js # BM25 keyword search via FTS5 - search/ - semantic.js # Vector similarity search - keyword.js # FTS5 BM25 search - hybrid.js # RRF fusion of semantic + keyword - strategies/ - structured.js # Structured text preparation - source.js # Raw source preparation -``` +**This pattern should be replicated for:** +- `complexity.js` → `src/complexity/rules/{language}.js` (same per-language rule pattern) +- `cfg.js` → `src/cfg/rules/{language}.js` (same per-language rule pattern) +- `dataflow.js` → `src/dataflow/extractors/{language}.js` (when more languages are supported) -The three search modes (semantic, keyword, hybrid) become composable search strategies rather than three code paths in one file. The store abstraction enables future pluggable backends (HNSW, DiskANN) without touching search logic. +The extractors refactoring proved the pattern works. Now apply it consistently. --- -## 9. parser.js Is No Longer a Monolith -- Downgrade Priority - -**Original analysis (S1):** 2,215 lines, 9 language extractors in one file. Highest priority. +## 9. ast.js — Stored Queryable AST Nodes -**Current state:** 404 lines. The native Rust engine now handles the heavy parsing. `parser.js` is a thin WASM fallback with `LANGUAGE_REGISTRY`, engine resolution, and minimal extraction. The extractors still exist but are much smaller per-language. +**Not in previous revision** — new module from PR #279. -**Revised recommendation:** This is no longer urgent. The Rust engine already implements the plugin system concept natively. The WASM path in `parser.js` at 404 lines is manageable. If the parser ever grows again (new languages added to WASM fallback), revisit -- but for now, this is fine. +**Current state:** 392 lines. Stores selected AST nodes during build for later querying: +- Node kinds: `call`, `new`, `string`, `regex`, `throw`, `await` +- Pattern matching via SQL GLOB with auto-wrapping +- Parent resolution via narrowest enclosing definition ---- - -## 10. The Native/WASM Abstraction -- Less Critical Now - -**Original analysis (S6):** Scattered `engine.name === 'native'` branching across multiple files. +**Architectural assessment:** This is a well-scoped module. At 392 lines it's appropriately sized. It follows the dual-function pattern (`astQueryData()` / `astQuery()`) but is otherwise clean. -**Current state:** The native engine is the primary path. WASM is a fallback. The branching still exists but is less problematic because most users never hit the WASM path. The unified engine interface is still the right design but it's a polish item, not a structural problem. - -**Revised priority:** Low-Medium. Do it when touching these files for other reasons. +**The main concern** is that AST node extraction during build overlaps with what `dataflow.js` and `cfg.js` also do — all three walk the AST. With the unified AST analysis framework proposed in item #6, a single AST walk could populate all three subsystems in one pass. --- -## 11. Qualified Names + Hierarchical Scoping -- Still Important +## 10. viewer.js at 948 Lines — Self-Contained but Bloated -**Original analysis (S13):** Flat node model with name collisions resolved by heuristics. +**Not in previous revision** — new module from PR #268. -**Current state:** Unchanged. The `nodes` table still has `(name, kind, file, line)` with no scope or qualified name. The `structure.js` module added `role` classification but not scoping. With the codebase now handling more complex analysis (communities, boundaries, flow tracing), the lack of qualified names creates more ambiguity in more places. +**Current state:** 948 lines generating self-contained interactive HTML with vis-network. Features: layout switching, physics toggle, search, color/size/cluster overlays, drill-down, detail panel, community detection. -**Ideal enhancement** (unchanged): +**Architectural assessment:** +- Embeds ALL node/edge data as JSON in the HTML — scales poorly for large graphs +- Client-side filtering only — no server-side optimization +- Hardcoded thresholds (fanIn >= 10, MI < 40) not derived from distribution +- Tight vis-network coupling — custom clustering logic deeply integrated +- Good: configuration cascading via `.plotDotCfg` with deep merge -```sql -ALTER TABLE nodes ADD COLUMN qualified_name TEXT; -- 'DateHelper.format' -ALTER TABLE nodes ADD COLUMN scope TEXT; -- 'DateHelper' -ALTER TABLE nodes ADD COLUMN visibility TEXT; -- 'public' | 'private' | 'protected' -``` +**This module is isolated** — it has minimal impact on the rest of the architecture. The main risk is HTML size growth for large codebases. --- -## 12. Domain Error Hierarchy -- More Urgent with 35 Modules +## 11. Qualified Names + Hierarchical Scoping — Partially Addressed -**Original analysis (S17):** Inconsistent error handling across ~12 modules. +**Previous state:** Flat node model with no scope or parent information. -**Current state:** 35 modules with inconsistent error handling. Some throw, some return null, some `logger.warn()` and continue, some `process.exit(1)`. The MCP server wraps everything in generic try-catch. The `check.js` module returns structured pass/fail objects but other modules don't. +**Current state:** Partially addressed via PR #270: +- `parent_id` column added to `nodes` table (migration v11) +- `contains` edges track parent-child relationships +- `parameter_of` edges link parameters to functions +- `childrenData()` query returns child symbols +- Extended kinds (`parameter`, `property`, `constant`) model sub-declarations -**`check.js` already demonstrates the right pattern** -- structured result objects with clear pass/fail semantics. This should be generalized: +**What's still missing:** +- `qualified_name` column (e.g., `DateHelper.format`) +- `scope` column (e.g., `DateHelper`) +- `visibility` column (`public`/`private`/`protected`) +- The `parent_id` FK only goes one level — deeply nested scopes (namespace > class > method > closure) aren't fully represented -```js -// errors.js -export class CodegraphError extends Error { - constructor(message, { code, file, cause } = {}) { - super(message) - this.code = code - this.file = file - this.cause = cause - } -} - -export class ParseError extends CodegraphError { code = 'PARSE_FAILED' } -export class DbError extends CodegraphError { code = 'DB_ERROR' } -export class ConfigError extends CodegraphError { code = 'CONFIG_INVALID' } -export class ResolutionError extends CodegraphError { code = 'RESOLUTION_FAILED' } -export class EngineError extends CodegraphError { code = 'ENGINE_UNAVAILABLE' } -export class AnalysisError extends CodegraphError { code = 'ANALYSIS_FAILED' } -export class BoundaryError extends CodegraphError { code = 'BOUNDARY_VIOLATION' } -``` +**Revised priority:** Medium → Low-Medium. The `parent_id` + `contains` edges solve the 80% case (class methods, interface members, struct fields). The remaining 20% (qualified names, deep nesting) is a polish item. --- -## 13. Public API Surface -- 120+ Exports Is Unsustainable - -**Original analysis (S18):** ~40 re-exports, no distinction between public and internal. - -**Current state:** 120+ exports from `index.js`. Every `*Data()` function, every CLI display function, every constant, every utility is exported. The public API is the entire internal surface. +## 12. builder.js at 1,355 Lines — Pipeline Now Has 7+ Opt-In Stages -**The problem is now 3x worse** and directly blocks any refactoring -- every internal rename could break an unnamed consumer. +**Previous state:** 1,173 lines with complexity as the only opt-in stage. -**Ideal architecture** (reinforced): +**Current state:** 1,355 lines. The build pipeline now has 4 opt-in stages: -```js -// index.js -- curated public API (~30 exports) -// Build -export { buildGraph } from './builder.js' - -// Analysis (data functions only -- no CLI formatters) -export { queryNameData, impactAnalysisData, fileDepsData, fnDepsData, - fnImpactData, diffImpactData, moduleMapData, statsData, - contextData, explainData, whereData, listFunctionsData, - rolesData } from './analysis/index.js' - -// New analysis modules -export { auditData } from './commands/audit.js' -export { checkData } from './commands/check.js' -export { complexityData } from './commands/complexity.js' -export { manifestoData } from './commands/manifesto.js' -export { triageData } from './commands/triage.js' -export { flowData } from './commands/flow.js' -export { communitiesData } from './commands/communities.js' - -// Search -export { searchData, hybridSearchData, embedSymbols } from './embeddings/index.js' - -// Infrastructure -export { detectCycles } from './analysis/cycles.js' -export { exportGraph } from './export.js' -export { startMcpServer } from './mcp/server.js' -export { loadConfig } from './config.js' - -// Constants -export { SYMBOL_KINDS, ALL_SYMBOL_KINDS } from './shared/constants.js' ``` - -Lock it with `package.json` exports: - -```json -{ - "exports": { - ".": "./src/index.js", - "./cli": "./src/cli.js" - } -} +Core pipeline (always): + collectFiles → detectChanges → parseFiles → insertNodes → + resolveImports → buildCallEdges → buildClassEdges → + resolveBarrels → insertEdges → buildStructure → classifyRoles + +Opt-in stages: + --complexity → computeComplexity() + --dataflow → buildDataflowEdges() (dynamic import) + --cfg → buildCFGData() (dynamic import) + AST nodes → extractASTNodes() (always, post-parse) ``` ---- - -## 14. Structure + Cochange + Communities -- Parallel Graph Models Need Unification +**Positive development:** The opt-in stages use dynamic imports — `dataflow.js` and `cfg.js` are only loaded when their flags are passed. This keeps default builds fast. -**Not in original analysis** -- these modules didn't exist. +**The pipeline architecture from the previous revision is even more relevant now.** Seven core stages + 4 opt-in stages = 11 total. Each should be independently testable with the pipeline runner handling transactions, logging, progress, and statistics. -**Current state:** Three separate analytical subsystems each build their own graph representation: +--- -- **`structure.js`** (668 lines): Builds directory nodes, computes cohesion/density/coupling metrics, classifies roles (entry, core, utility, adapter, leaf, dead). Has its own BFS and metrics computation. -- **`cochange.js`** (502 lines): Builds temporal coupling graph from git history. Stores in `co_changes` table with Jaccard coefficients. Independent of the dependency graph. -- **`communities.js`** (310 lines): Uses graphology to build an in-memory graph from edges, runs Louvain community detection, computes modularity and drift. +## 13. Export Formats — 6 Formats, Well-Contained -Each constructs its own graph representation independently. There's no shared graph abstraction they all operate on. +**Previous state:** DOT, Mermaid, JSON. -**Ideal architecture -- unified graph model:** +**Current state:** DOT, Mermaid, JSON, GraphML, GraphSON, Neo4j CSV. Export.js at 681 lines (unchanged — the new formats were already counted in the previous revision). -``` -src/ - graph/ - model.js # In-memory graph representation (nodes + edges + metadata) - builders/ - dependency.js # Build from SQLite edges (imports, calls, extends) - structure.js # Build from file/directory hierarchy - temporal.js # Build from git history (co-changes) - algorithms/ - bfs.js # Breadth-first traversal (used by impact, flow, etc.) - shortest-path.js # Path finding (used by path command) - tarjan.js # Cycle detection (currently in cycles.js) - louvain.js # Community detection (currently uses graphology) - centrality.js # Fan-in/fan-out, betweenness (used by triage, hotspots) - clustering.js # Cohesion, coupling, density metrics - classifiers/ - roles.js # Node role classification - risk.js # Risk scoring (currently in triage.js) -``` - -The graph model is a shared in-memory structure that multiple builders can populate and multiple algorithms can query. This eliminates the repeated graph construction across modules and makes algorithms composable -- you can run community detection on the dependency graph, the temporal graph, or a merged graph. +**Assessment:** Well-contained. The export module adds formats without affecting other modules. No architectural concerns. --- -## 15. Pagination Pattern Needs Standardization - -**Not in original analysis** -- paginate.js was just introduced. +## 14. Constants Hierarchy — A Good Foundation -**Current state:** `paginate.js` (106 lines) provides `paginate()` and `paginateResult()` helpers plus `MCP_DEFAULTS` with per-command limits. But each module integrates pagination differently -- some pass `opts` to paginate, some manually slice arrays, some use `LIMIT/OFFSET` in SQL, some paginate in memory after fetching all results. +**Not in previous revision** — introduced across PRs #267, #270, #279. -**Ideal architecture:** Pagination belongs in the repository layer (SQL `LIMIT/OFFSET`) for data fetching and in the command runner for result shaping. The current pattern of fetching all data then slicing in memory doesn't scale. The repository should accept pagination parameters directly: +**Current state:** Three-tiered constants in `queries.js`: ```js -// In repository -findNodes(filters, { limit, offset, orderBy }) { - // Generates SQL with LIMIT/OFFSET -- never fetches more than needed -} - -// In command runner (after execute) -runner.paginate(result, 'functions', opts) // Consistent shaping for all commands +// Symbol kinds +CORE_SYMBOL_KINDS = ['function', 'method', 'class', 'interface', 'type', + 'struct', 'enum', 'trait', 'record', 'module'] +EXTENDED_SYMBOL_KINDS = ['parameter', 'property', 'constant'] +EVERY_SYMBOL_KIND = [...CORE_SYMBOL_KINDS, ...EXTENDED_SYMBOL_KINDS] +ALL_SYMBOL_KINDS = CORE_SYMBOL_KINDS // backward compat alias + +// Edge kinds +CORE_EDGE_KINDS = ['imports', 'imports-type', 'reexports', 'calls', 'extends', 'implements'] +STRUCTURAL_EDGE_KINDS = ['parameter_of', 'receiver'] +EVERY_EDGE_KIND = [...CORE_EDGE_KINDS, ...STRUCTURAL_EDGE_KINDS] + +// AST node kinds (in ast.js) +AST_NODE_KINDS = ['call', 'new', 'string', 'regex', 'throw', 'await'] ``` ---- - -## 16. Testing -- Good Coverage, Wrong Distribution - -**Original analysis (S11):** Missing proper unit tests. +**This is well-designed.** The tiered approach lets older code use `ALL_SYMBOL_KINDS` (10 core kinds) while new code can opt into `EVERY_SYMBOL_KIND` (13 kinds). The `contains` edge is stored in the `edges` table but excluded from coupling metrics via the `STRUCTURAL_EDGE_KINDS` distinction. -**Current state:** 59 test files -- major improvement. Tests exist across: -- `tests/unit/` -- 18 files -- `tests/integration/` -- 18 files -- `tests/parsers/` -- 8 files -- `tests/engines/` -- 2 files (parity tests) -- `tests/search/` -- 3 files -- `tests/incremental/` -- 2 files - -**What's still missing:** -- Unit tests for pure graph algorithms (BFS, Tarjan) in isolation -- Unit tests for confidence scoring with various inputs -- Unit tests for the triage risk scoring formula -- Mock-based tests (the repository pattern would enable `InMemoryRepository`) -- Many "unit" tests still hit SQLite -- they're integration tests in the unit directory - -The test count is adequate. The issue is that without the repository pattern, true unit testing is impossible for most modules -- they all need a real SQLite DB. +**One concern:** These constants are scattered across multiple files (`queries.js`, `ast.js`). They should all live in a single `shared/constants.js` as proposed in item #3. --- -## 17. Event-Driven Pipeline -- Still Relevant for Scale +## Updated Priority Summary -**Original analysis (S7):** Batch pipeline with no progress reporting. +### Items That Improved Since Last Revision -**Current state:** Still batch. The `change-journal.js` module adds NDJSON event logging for watch mode, which is a step toward events -- but the build pipeline itself is still synchronous batch. For repos with 10K+ files, users still see no progress during builds. +| # | Item | What improved | +|---|------|--------------| +| 9 | Parser plugin system (was #20) | Extractors split into `src/extractors/` — **done** | +| 11 | Qualified names (was #12) | `parent_id`, `contains` edges, `parameter_of` — **partially done** | +| 5 | CLI surface area (was #5) | 5 commands consolidated in PR #280 — **started** | +| 3 | Constants organization (was part of #3) | Tiered `CORE_`/`EXTENDED_`/`EVERY_` hierarchy — **started** | +| -- | normalizeSymbol (new) | Stable JSON schema utility — **done** | -**Ideal architecture** (unchanged, lower priority than structural issues): +### Items That Worsened Since Last Revision -```js -pipeline.on('file:parsed', (file, symbols) => { /* progress */ }) -pipeline.on('file:indexed', (file, nodeCount) => { /* progress */ }) -pipeline.on('build:complete', (stats) => { /* summary */ }) -await pipeline.run(rootDir) -``` +| # | Item | What worsened | +|---|------|--------------| +| 1 | Dual-function pattern | 15 → 19 modules | +| 2 | Repository pattern | 9 → 13 tables, 20 → 25+ modules with raw SQL | +| 3 | queries.js size | 3,110 → 3,395 lines | +| 4 | MCP monolith | 25 → 34 tools in one file | +| 5 | CLI size | 1,285 → 1,557 lines (despite consolidation) | +| 6 | Public API | 120+ → 140+ exports | +| 8 | AST analysis duplication | 1 module (complexity) → 3 modules (+ cfg, dataflow) with parallel rule engines | --- -## 18. Dead Symbol Cleanup -- 27% of Classified Code Is Unused - -**Not in original analysis** -- the `roles` classification that surfaces dead symbols didn't exist yet. +## Revised Summary — Priority Ordering by Architectural Impact -**Current state:** Codegraph's own role classification reports 221 dead symbols -- 27% of all classified code. In a project this young (~10 days old at time of measurement), a quarter of the symbols being unused signals systematic overproduction: speculative helpers, leftover refactoring artifacts, and the dual-function pattern generating display functions that nothing calls. - -**Root causes:** -- The `*Data()` / `*()` dual-function pattern (Section 1) means every data function has a display counterpart. MCP and programmatic consumers only call `*Data()`, leaving many `*()` functions uncalled -- `index.js` exports 120+ symbols (Section 13) with no consumer tracking -- functions are exported "just in case" -- Rapid feature addition without pruning -- each new module adds helpers that may only be used during development - -**Ideal approach -- continuous dead code hygiene:** - -1. **Audit pass:** Run `codegraph roles --role dead -T` and categorize results: - - **Truly dead:** Remove immediately (unused helpers, orphaned formatters) - - **Entry points:** CLI handlers, MCP tool handlers, test utilities -- mark as `@entry` or add to a known-entries list so the classifier doesn't flag them - - **Public API:** Exported but uncalled internally -- decide if they're part of the supported API or remove from `index.js` - -2. **CI gate:** Add a dead-symbol threshold to `manifesto.js` rules: - ```json - { - "rule": "max-dead-ratio", - "warn": 0.15, - "fail": 0.25, - "message": "Dead symbol ratio exceeds {threshold}" - } - ``` - -3. **Prevention:** The Command/Query separation (Section 1) and curated API surface (Section 13) eliminate the two biggest dead-code factories. Once display functions are internal to the CLI layer and exports are curated, new dead code becomes visible immediately. - -**Target:** Reduce dead symbol ratio from 27% to under 10%. - ---- - -## 19. Community Drift -- 40% of Files Are in the Wrong Logical Module - -**Not in original analysis** -- `communities.js` didn't exist yet. - -**Current state:** Louvain community detection on the dependency graph finds that 40% of files belong to a different logical community than their directory suggests. This means the file organization actively misleads developers about which modules are coupled. - -**What drift means concretely:** -- Files in `src/` root that should be grouped (e.g., `triage.js`, `audit.js`, `manifesto.js` form a "code health" community but live alongside unrelated modules) -- Utility functions in domain modules that are actually shared infrastructure -- Tight coupling between files in different conceptual areas (e.g., `structure.js` and `queries.js` are more coupled to each other than to their neighbors) - -**Ideal approach -- align directory structure to communities:** - -1. **Measure baseline:** `codegraph communities -T` to get current modularity score and drift percentage -2. **Map communities to directories:** The restructuring proposed in Sections 1, 3, 4, 5 would naturally create directories that match logical communities: - ``` - src/ - analysis/ # Community: query/impact/context/explain/roles - commands/ # Community: CLI-specific formatting - health/ # Community: audit/triage/manifesto/check/complexity - graph/ # Community: structure/communities/cochange/cycles - infrastructure/ # Community: db/pagination/config/logger - ``` -3. **Track drift as a metric:** Add modularity score and drift percentage to `stats` output. Regressing drift should trigger a warning. -4. **CI gate:** Add a drift threshold to `manifesto.js`: - ```json - { - "rule": "max-community-drift", - "warn": 0.30, - "fail": 0.45, - "message": "Community drift exceeds {threshold}" - } - ``` - -**Target:** Reduce drift from 40% to under 20% through directory restructuring. +| # | Change | Impact | Category | Previous # | +|---|--------|--------|----------|------------| +| **1** | **Command/Query separation — eliminate dual-function pattern across 19 modules** | **Critical** | Separation of concerns | #1 (15→19 modules) | +| **2** | **Repository pattern for data access — raw SQL in 25+ modules, 13 tables** | **Critical** | Testability, maintainability | #2 (9→13 tables) | +| **3** | **Decompose queries.js (3,395 lines) into analysis modules + shared constants** | **Critical** | Modularity | #3 (3,110→3,395) | +| **4** | **Unified AST analysis framework — complexity + CFG + dataflow share no infrastructure** | **Critical** | Code duplication | New (3 modules, ~4.8K lines, parallel rule engines) | +| **5** | **Composable MCP tool registry (34 tools in 1,370 lines)** | **High** | Extensibility | #4 (25→34 tools) | +| **6** | **CLI command objects (47 commands in 1,557 lines)** | **High** | Maintainability | #5 (45→47 commands, consolidation started) | +| **7** | **Curated public API surface (140+ to ~35 exports)** | **High** | API stability | #6 (120→140+ exports) | +| **8** | **Domain error hierarchy (50 modules, inconsistent handling)** | **High** | Reliability | #7 (35→50 modules) | +| **9** | **Builder pipeline architecture (1,355 lines, 11 stages, 4 opt-in)** | **High** | Testability, reuse | #9 (1,173→1,355, +2 opt-in stages) | +| **10** | **Embedder subsystem (1,113 lines, 3 search engines)** | **Medium-High** | Extensibility | #10 (unchanged) | +| **11** | **Unified graph model for structure/cochange/communities/viewer** | **Medium-High** | Cohesion | #11 (viewer now also builds its own graph) | +| **12** | **Pagination standardization (SQL-level + command runner)** | **Medium** | Consistency | #13 (unchanged) | +| **13** | **Testing pyramid with InMemoryRepository** | **Medium** | Quality | #14 (59→70 test files, same DB coupling) | +| **14** | **Event-driven pipeline for streaming** | **Medium** | Scalability, UX | #15 (unchanged) | +| **15** | **Qualified names (remaining: qualified_name, scope, visibility columns)** | **Low-Medium** | Data model | #12 (partially addressed by parent_id) | +| **16** | **Query result caching (34 MCP tools)** | **Low-Medium** | Performance | #16 (25→34 tools) | +| **17** | **Unified engine interface (Strategy)** | **Low-Medium** | Abstraction | #17 (unchanged) | +| **18** | **Subgraph export with filtering** | **Low-Medium** | Usability | #18 (unchanged) | +| **19** | **Transitive import-aware confidence** | **Low** | Accuracy | #19 (unchanged) | +| **20** | **Config profiles for monorepos** | **Low** | Feature | #21 (unchanged) | + +### Items Resolved / Downgraded + +| Previous # | Item | Status | +|------------|------|--------| +| #20 | Parser plugin system | **Resolved** — extractors split into `src/extractors/` | +| #8 | Decompose complexity.js (standalone) | **Subsumed** by new #4 (unified AST analysis framework) | --- -## 20. Function-Level Cycles -- 9 Circular Dependencies - -**Not in original analysis** -- cycle detection existed but function-level cycles weren't measured. - -**Current state:** `codegraph cycles` reports 9 function-level circular dependencies. While the codebase has no file-level cycles (imports are acyclic), function call graphs contain mutual recursion and indirect loops. - -**Why this matters:** -- Circular call chains make impact analysis unreliable -- a change to any function in a cycle potentially affects all others -- They complicate the proposed decomposition (Sections 1, 3) -- you can't cleanly split modules if their functions are mutually dependent -- They indicate hidden coupling that the module structure doesn't reveal +## New Architectural Concern: Three Independent AST Rule Engines -**Ideal approach:** +The most significant architectural development since the last revision is the emergence of **three independent AST analysis modules** that share the same fundamental pattern but no infrastructure: -1. **Identify and classify:** Run `codegraph cycles` and categorize each cycle: - - **Intentional recursion:** Mutual recursion in tree walkers, AST visitors -- document with comments, exclude from CI gates - - **Accidental coupling:** Function A calls B which calls C which calls A -- these need refactoring - - **Layering violations:** A query function calling a builder function that calls back into queries -- break by introducing an interface boundary +| Module | Lines | Languages | Pattern | +|--------|-------|-----------|---------| +| `complexity.js` | 2,163 | 8 | Per-language rules map → AST walk → collect metrics | +| `cfg.js` | 1,451 | 9 | Per-language rules map → AST walk → build basic blocks | +| `dataflow.js` | 1,187 | 1 (JS/TS) | Scope stack → AST walk → collect flows | -2. **Break accidental cycles:** - - **Extract shared logic:** If A and B both need the same computation, extract it to a third function that both call - - **Invert dependencies:** If a low-level function calls a high-level one, pass the needed data as a parameter instead - - **Event/callback:** For unavoidable bidirectional communication, use callbacks or events instead of direct calls +Total: **4,801 lines** of parallel AST walking implementations. All three: +- Walk function-level ASTs from tree-sitter parse trees +- Use language-specific rule maps keyed by AST node type +- Build intermediate data structures during the walk +- Write results to dedicated DB tables +- Provide query functions + CLI formatters -3. **CI gate:** Add to `check.js` predicates: - ```json - { - "rule": "no-new-cycles", - "scope": "function", - "message": "New function-level cycle introduced: {cycle}" - } - ``` +Additionally, `ast.js` (392 lines) does a fourth AST walk to extract stored nodes. -4. **Prevention:** The layered architecture proposed throughout this document (analysis → infrastructure → db) naturally prevents cycles -- lower layers never import from higher layers. +**The extractors refactoring showed the path:** split per-language rules into files, share the engine. `cfg.js` already took a step in this direction with `makeCfgRules(overrides)` — a factory function for language-specific CFG rules with defaults. Apply this pattern to all four AST analysis passes: -**Target:** Reduce from 9 cycles to 0 accidental cycles (intentional recursion documented and exempted). - ---- - -## Remaining Items (Unchanged from Original) - -- **Config profiles (S8):** Single flat config, no monorepo profiles. Still relevant but not blocking anything. -- **Transitive import-aware confidence (S9):** Walk import graph before falling back to proximity heuristics. Targeted algorithmic improvement. -- **Query result caching (S14):** LRU/TTL cache between analysis and repository. More valuable now with 25 MCP tools. -- **Subgraph export filtering (S16):** Export the full graph or nothing. Still relevant for usability. - ---- - -## Revised Summary -- Priority Ordering by Architectural Impact +``` +src/ + ast-analysis/ + visitor.js # Shared AST visitor with hook points + rules/ + complexity/{lang}.js # Cognitive/cyclomatic rules + cfg/{lang}.js # Basic-block rules + dataflow/{lang}.js # Define-use chain rules + ast-store/{lang}.js # Node extraction rules + engine.js # Single-pass or multi-pass orchestrator +``` -| # | Change | Impact | Category | Original # | -|---|--------|--------|----------|------------| -| **1** | **Command/Query separation -- eliminate dual-function pattern across 15 modules** | **Critical** | Separation of concerns | S3 (was High) | -| **2** | **Repository pattern for data access -- SQL in 20+ modules** | **Critical** | Testability, maintainability | S2 (was High) | -| **3** | **Decompose queries.js (3,110 lines) into analysis modules** | **Critical** | Modularity | S3 (was High) | -| **4** | **Composable MCP tool registry (25 tools in 1,212 lines)** | **High** | Extensibility | S10 (was Medium) | -| **5** | **CLI command objects (45 commands in 1,285 lines)** | **High** | Maintainability | S12 (was Medium) | -| **6** | **Curated public API surface (120+ to ~30 exports)** | **High** | API stability | S18 (was Medium) | -| **7** | **Domain error hierarchy (35 modules, inconsistent handling)** | **High** | Reliability | S17 (was Medium) | -| **8** | **Decompose complexity.js (2,163 lines) into rules/engine** | **High** | Modularity | New | -| **9** | **Builder pipeline architecture (1,173 lines)** | **High** | Testability, reuse | S4 (was High) | -| **10** | **Embedder subsystem (1,113 lines, 3 search engines)** | **Medium-High** | Extensibility | S5 (was Medium) | -| **11** | **Unified graph model for structure/cochange/communities** | **Medium-High** | Cohesion | New | -| **12** | **Qualified names + hierarchical scoping** | **Medium** | Data model accuracy | S13 (unchanged) | -| **13** | **Pagination standardization (SQL-level + command runner)** | **Medium** | Consistency | New | -| **14** | **Testing pyramid with InMemoryRepository** | **Medium** | Quality | S11 (unchanged) | -| **15** | **Event-driven pipeline for streaming** | **Medium** | Scalability, UX | S7 (unchanged) | -| **16** | **Query result caching (25 MCP tools)** | **Low-Medium** | Performance | S14 (unchanged) | -| **17** | **Dead symbol cleanup (27% dead code ratio)** | **Medium** | Code hygiene | New | -| **18** | **Reduce community drift (40% misplaced files)** | **Medium** | Cohesion | New | -| **19** | **Break function-level cycles (9 circular deps)** | **Medium** | Correctness | New | -| **20** | **Unified engine interface (Strategy)** | **Low-Medium** | Abstraction | S6 (was Medium-High) | -| **21** | **Subgraph export with filtering** | **Low-Medium** | Usability | S16 (unchanged) | -| **22** | **Transitive import-aware confidence** | **Low** | Accuracy | S9 (unchanged) | -| **23** | **Parser plugin system** | **Low** | Modularity | S1 (was High -- parser.js shrank to 404 lines) | -| **24** | **Config profiles for monorepos** | **Low** | Feature | S8 (unchanged) | - -**The structural priority shifted.** In the original analysis, the parser monolith was #1 -- it's now #23 because the native engine solved it. The new #1 is the command/query separation: the dual-function anti-pattern replicated across 15 modules is the single biggest source of code duplication and coupling in the codebase. Items 1-3 are the foundation -- they restructure the core and everything else becomes easier. Items 4-7 are high-impact but can be done in parallel. Items 8-10 are large-file decompositions that follow naturally once the shared infrastructure exists. Items 17-19 (dead symbols, community drift, function cycles) are health metrics that improve naturally as the structural changes land -- but also benefit from explicit CI gates to prevent regression. +A single AST walk with pluggable visitors would: +1. Eliminate 3 redundant tree traversals per function +2. Share language-specific node type mappings +3. Allow new analyses to plug in without creating another 1K+ line module +4. Enable the 4 opt-in build stages to share a single parse pass --- -*Revised 2026-03-02. Cold architectural analysis -- no implementation constraints applied.* +*Revised 2026-03-03. Cold architectural analysis — no implementation constraints applied.* From fa7eee8e0d2f6905d47a80dd8aa6de617d609835 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Tue, 3 Mar 2026 01:06:56 -0700 Subject: [PATCH 2/4] fix: correct extractor line counts and duplicate section numbering - Update extractor line counts from 2,299 to actual 3,023 (verified via wc -l) - Fix individual per-file counts in both ROADMAP.md and architecture.md - Remove duplicate section 3.8 numbering for strikethrough/subsumed item --- docs/roadmap/ROADMAP.md | 8 +++----- generated/architecture.md | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/docs/roadmap/ROADMAP.md b/docs/roadmap/ROADMAP.md index 163d1c9d..3c6286ad 100644 --- a/docs/roadmap/ROADMAP.md +++ b/docs/roadmap/ROADMAP.md @@ -439,8 +439,8 @@ Persist and query selected AST node types for pattern-based codebase exploration Split per-language extractors from monolithic `parser.js` into dedicated modules. -- ✅ New `src/extractors/` directory with 11 files (2,299 lines total) -- ✅ One file per language: `javascript.js` (750), `csharp.js` (248), `php.js` (243), `java.js` (230), `rust.js` (225), `ruby.js` (188), `go.js` (172), `python.js` (150), `hcl.js` (73) +- ✅ New `src/extractors/` directory with 11 files (3,023 lines total) +- ✅ One file per language: `javascript.js` (892), `csharp.js` (311), `php.js` (322), `java.js` (290), `rust.js` (295), `ruby.js` (277), `go.js` (237), `python.js` (284), `hcl.js` (95) - ✅ Shared utilities in `helpers.js` (`nodeEndLine()`, `findChild()`) - ✅ Barrel export via `index.js` - ✅ Consistent return schema: `{ definitions, calls, imports, classes, exports }` @@ -705,9 +705,7 @@ Export only `*Data()` functions (the command execute functions). Never export CL **Affected files:** `src/index.js`, `package.json` -### ~~3.8~~ -- ~~Decompose complexity.js~~ Subsumed by 3.1 - -> The standalone complexity decomposition from the previous revision is now part of the unified AST analysis framework (3.1). The `complexity.js` per-language rules become `ast-analysis/rules/complexity/{lang}.js` alongside CFG and dataflow rules. +> **Removed: Decompose complexity.js** — Subsumed by 3.1. The standalone complexity decomposition from the previous revision is now part of the unified AST analysis framework (3.1). The `complexity.js` per-language rules become `ast-analysis/rules/complexity/{lang}.js` alongside CFG and dataflow rules. ### 3.8 -- Domain Error Hierarchy diff --git a/generated/architecture.md b/generated/architecture.md index 3011a997..c803fb07 100644 --- a/generated/architecture.md +++ b/generated/architecture.md @@ -30,7 +30,7 @@ | Test files | 59 | 70 | +11 | | Node kinds | 10 | 13 | +3 (parameter, property, constant) | | Edge kinds | 6 | 9 | +3 (contains, parameter_of, receiver) | -| Extractor modules | 0 (inline in parser.js) | 11 files, 2,299 lines | New directory | +| Extractor modules | 0 (inline in parser.js) | 11 files, 3,023 lines | New directory | **Key patterns observed in this burst:** @@ -209,19 +209,19 @@ Both complexity and CFG analysis walk the same AST trees with language-specific **Previous state:** parser.js at 404 lines with inline extractors. -**Current state:** `src/extractors/` directory with 11 files totaling 2,299 lines: +**Current state:** `src/extractors/` directory with 11 files totaling 3,023 lines: | File | Lines | Language | |------|-------|----------| -| `javascript.js` | 750 | JS/TS/TSX | -| `csharp.js` | 248 | C# | -| `php.js` | 243 | PHP | -| `java.js` | 230 | Java | -| `rust.js` | 225 | Rust | -| `ruby.js` | 188 | Ruby | -| `go.js` | 172 | Go | -| `python.js` | 150 | Python | -| `hcl.js` | 73 | Terraform/HCL | +| `javascript.js` | 892 | JS/TS/TSX | +| `csharp.js` | 311 | C# | +| `php.js` | 322 | PHP | +| `java.js` | 290 | Java | +| `rust.js` | 295 | Rust | +| `ruby.js` | 277 | Ruby | +| `go.js` | 237 | Go | +| `python.js` | 284 | Python | +| `hcl.js` | 95 | Terraform/HCL | | `helpers.js` | 11 | Shared utilities | | `index.js` | 9 | Barrel export | From cbda26682ed3660fc0119efbb51dfef02e14c54f Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Tue, 3 Mar 2026 01:28:17 -0700 Subject: [PATCH 3/4] refactor: consolidate MCP tools to match CLI changes from PR #280 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove 3 redundant MCP tools and add 1 new tool to mirror the CLI consolidation done in PR #280: - Remove `explain` tool → fold into `audit` with `quick` param - Remove `hotspots` tool → fold into `triage` with `level` param - Remove `manifesto` tool → fold into `check` with manifesto-mode routing - Add standalone `path` tool (was only accessible via query mode=path) All backing data functions are unchanged. The `query` tool still supports mode=path for backward compatibility. Impact: 1 functions changed, 1 affected --- src/mcp.js | 245 ++++++++++++++++++++++------------------- tests/unit/mcp.test.js | 48 ++++++-- 2 files changed, 168 insertions(+), 125 deletions(-) diff --git a/src/mcp.js b/src/mcp.js index e965a4a8..c1ad7a5c 100644 --- a/src/mcp.js +++ b/src/mcp.js @@ -70,6 +70,27 @@ const BASE_TOOLS = [ required: ['name'], }, }, + { + name: 'path', + description: 'Find shortest path between two symbols in the dependency graph', + inputSchema: { + type: 'object', + properties: { + from: { type: 'string', description: 'Source symbol name' }, + to: { type: 'string', description: 'Target symbol name' }, + depth: { type: 'number', description: 'Max traversal depth (default: 10)' }, + edge_kinds: { + type: 'array', + items: { type: 'string', enum: EVERY_EDGE_KIND }, + description: 'Edge kinds to follow (default: ["calls"])', + }, + from_file: { type: 'string', description: 'Disambiguate source by file' }, + to_file: { type: 'string', description: 'Disambiguate target by file' }, + no_tests: { type: 'boolean', description: 'Exclude test files', default: false }, + }, + required: ['from', 'to'], + }, + }, { name: 'file_deps', description: 'Show what a file imports and what imports it', @@ -207,20 +228,6 @@ const BASE_TOOLS = [ required: ['name'], }, }, - { - name: 'explain', - description: - 'Structural summary of a file or function: public/internal API, data flow, dependencies. No LLM needed.', - inputSchema: { - type: 'object', - properties: { - target: { type: 'string', description: 'File path or function name' }, - no_tests: { type: 'boolean', description: 'Exclude test files', default: false }, - ...PAGINATION_PROPS, - }, - required: ['target'], - }, - }, { name: 'where', description: @@ -357,29 +364,6 @@ const BASE_TOOLS = [ }, }, }, - { - name: 'hotspots', - description: - 'Find structural hotspots: files or directories with extreme fan-in, fan-out, or symbol density', - inputSchema: { - type: 'object', - properties: { - metric: { - type: 'string', - enum: ['fan-in', 'fan-out', 'density', 'coupling'], - description: 'Metric to rank by', - }, - level: { - type: 'string', - enum: ['file', 'directory'], - description: 'Rank files or directories', - }, - limit: { type: 'number', description: 'Number of results to return', default: 10 }, - no_tests: { type: 'boolean', description: 'Exclude test files', default: false }, - offset: { type: 'number', description: 'Skip this many results (pagination, default: 0)' }, - }, - }, - }, { name: 'co_changes', description: @@ -469,23 +453,6 @@ const BASE_TOOLS = [ }, }, }, - { - name: 'manifesto', - description: - 'Evaluate manifesto rules and return pass/fail verdicts for code health. Checks function complexity, file metrics, and cycle rules against configured thresholds.', - inputSchema: { - type: 'object', - properties: { - file: { type: 'string', description: 'Scope to file (partial match)' }, - no_tests: { type: 'boolean', description: 'Exclude test files', default: false }, - kind: { - type: 'string', - description: 'Filter by symbol kind (function, method, class, etc.)', - }, - ...PAGINATION_PROPS, - }, - }, - }, { name: 'communities', description: @@ -543,6 +510,11 @@ const BASE_TOOLS = [ type: 'object', properties: { target: { type: 'string', description: 'File path or function name' }, + quick: { + type: 'boolean', + description: 'Structural summary only (skip impact + health)', + default: false, + }, depth: { type: 'number', description: 'Impact analysis depth (default: 3)', default: 3 }, file: { type: 'string', description: 'Scope to file (partial match)' }, kind: { @@ -550,6 +522,7 @@ const BASE_TOOLS = [ description: 'Filter by symbol kind (function, method, class, etc.)', }, no_tests: { type: 'boolean', description: 'Exclude test files', default: false }, + ...PAGINATION_PROPS, }, required: ['target'], }, @@ -607,6 +580,12 @@ const BASE_TOOLS = [ inputSchema: { type: 'object', properties: { + level: { + type: 'string', + enum: ['function', 'file', 'directory'], + description: + 'Granularity: function (default) | file | directory. File/directory shows hotspots', + }, sort: { type: 'string', enum: ['risk', 'complexity', 'churn', 'fan-in', 'mi'], @@ -701,12 +680,16 @@ const BASE_TOOLS = [ { name: 'check', description: - 'Run CI validation predicates against git changes. Checks for new cycles, blast radius violations, signature changes, and boundary violations. Returns pass/fail per predicate — ideal for CI gates.', + 'CI gate: run manifesto rules (no args), diff predicates (with ref/staged), or both (with rules flag). Returns pass/fail verdicts.', inputSchema: { type: 'object', properties: { ref: { type: 'string', description: 'Git ref to diff against (default: HEAD)' }, staged: { type: 'boolean', description: 'Analyze staged changes instead of unstaged' }, + rules: { + type: 'boolean', + description: 'Also run manifesto rules alongside diff predicates', + }, cycles: { type: 'boolean', description: 'Enable cycles predicate (default: true)' }, blast_radius: { type: 'number', @@ -715,7 +698,13 @@ const BASE_TOOLS = [ signatures: { type: 'boolean', description: 'Enable signatures predicate (default: true)' }, boundaries: { type: 'boolean', description: 'Enable boundaries predicate (default: true)' }, depth: { type: 'number', description: 'Max BFS depth for blast radius (default: 3)' }, + file: { type: 'string', description: 'Scope to file (partial match, manifesto mode)' }, + kind: { + type: 'string', + description: 'Filter by symbol kind (manifesto mode)', + }, no_tests: { type: 'boolean', description: 'Exclude test files', default: false }, + ...PAGINATION_PROPS, }, }, }, @@ -894,6 +883,15 @@ export async function startMCPServer(customDbPath, options = {}) { } break; } + case 'path': + result = pathData(args.from, args.to, dbPath, { + maxDepth: args.depth ?? 10, + edgeKinds: args.edge_kinds, + fromFile: args.from_file, + toFile: args.to_file, + noTests: args.no_tests, + }); + break; case 'file_deps': result = fileDepsData(args.file, dbPath, { noTests: args.no_tests, @@ -956,13 +954,6 @@ export async function startMCPServer(customDbPath, options = {}) { offset: args.offset ?? 0, }); break; - case 'explain': - result = explainData(args.target, dbPath, { - noTests: args.no_tests, - limit: Math.min(args.limit ?? MCP_DEFAULTS.explain, MCP_MAX_LIMIT), - offset: args.offset ?? 0, - }); - break; case 'where': result = whereData(args.target, dbPath, { file: args.file_mode, @@ -1132,17 +1123,6 @@ export async function startMCPServer(customDbPath, options = {}) { }); break; } - case 'hotspots': { - const { hotspotsData } = await import('./structure.js'); - result = hotspotsData(dbPath, { - metric: args.metric, - level: args.level, - limit: Math.min(args.limit ?? MCP_DEFAULTS.hotspots, MCP_MAX_LIMIT), - offset: args.offset ?? 0, - noTests: args.no_tests, - }); - break; - } case 'co_changes': { const { coChangeData, coChangeTopData } = await import('./cochange.js'); result = args.file @@ -1200,17 +1180,6 @@ export async function startMCPServer(customDbPath, options = {}) { }); break; } - case 'manifesto': { - const { manifestoData } = await import('./manifesto.js'); - result = manifestoData(dbPath, { - file: args.file, - noTests: args.no_tests, - kind: args.kind, - limit: Math.min(args.limit ?? MCP_DEFAULTS.manifesto, MCP_MAX_LIMIT), - offset: args.offset ?? 0, - }); - break; - } case 'communities': { const { communitiesData } = await import('./communities.js'); result = communitiesData(dbPath, { @@ -1235,13 +1204,21 @@ export async function startMCPServer(customDbPath, options = {}) { break; } case 'audit': { - const { auditData } = await import('./audit.js'); - result = auditData(args.target, dbPath, { - depth: args.depth, - file: args.file, - kind: args.kind, - noTests: args.no_tests, - }); + if (args.quick) { + result = explainData(args.target, dbPath, { + noTests: args.no_tests, + limit: Math.min(args.limit ?? MCP_DEFAULTS.explain, MCP_MAX_LIMIT), + offset: args.offset ?? 0, + }); + } else { + const { auditData } = await import('./audit.js'); + result = auditData(args.target, dbPath, { + depth: args.depth, + file: args.file, + kind: args.kind, + noTests: args.no_tests, + }); + } break; } case 'batch_query': { @@ -1255,18 +1232,30 @@ export async function startMCPServer(customDbPath, options = {}) { break; } case 'triage': { - const { triageData } = await import('./triage.js'); - result = triageData(dbPath, { - sort: args.sort, - minScore: args.min_score, - role: args.role, - file: args.file, - kind: args.kind, - noTests: args.no_tests, - weights: args.weights, - limit: Math.min(args.limit ?? MCP_DEFAULTS.triage, MCP_MAX_LIMIT), - offset: args.offset ?? 0, - }); + if (args.level === 'file' || args.level === 'directory') { + const { hotspotsData } = await import('./structure.js'); + const metric = args.sort === 'risk' ? 'fan-in' : args.sort; + result = hotspotsData(dbPath, { + metric, + level: args.level, + limit: Math.min(args.limit ?? MCP_DEFAULTS.hotspots, MCP_MAX_LIMIT), + offset: args.offset ?? 0, + noTests: args.no_tests, + }); + } else { + const { triageData } = await import('./triage.js'); + result = triageData(dbPath, { + sort: args.sort, + minScore: args.min_score, + role: args.role, + file: args.file, + kind: args.kind, + noTests: args.no_tests, + weights: args.weights, + limit: Math.min(args.limit ?? MCP_DEFAULTS.triage, MCP_MAX_LIMIT), + offset: args.offset ?? 0, + }); + } break; } case 'branch_compare': { @@ -1321,17 +1310,45 @@ export async function startMCPServer(customDbPath, options = {}) { break; } case 'check': { - const { checkData } = await import('./check.js'); - result = checkData(dbPath, { - ref: args.ref, - staged: args.staged, - cycles: args.cycles, - blastRadius: args.blast_radius, - signatures: args.signatures, - boundaries: args.boundaries, - depth: args.depth, - noTests: args.no_tests, - }); + const isDiffMode = args.ref || args.staged; + + if (!isDiffMode && !args.rules) { + // No ref, no staged → run manifesto rules on whole codebase + const { manifestoData } = await import('./manifesto.js'); + result = manifestoData(dbPath, { + file: args.file, + noTests: args.no_tests, + kind: args.kind, + limit: Math.min(args.limit ?? MCP_DEFAULTS.manifesto, MCP_MAX_LIMIT), + offset: args.offset ?? 0, + }); + } else { + const { checkData } = await import('./check.js'); + const checkResult = checkData(dbPath, { + ref: args.ref, + staged: args.staged, + cycles: args.cycles, + blastRadius: args.blast_radius, + signatures: args.signatures, + boundaries: args.boundaries, + depth: args.depth, + noTests: args.no_tests, + }); + + if (args.rules) { + const { manifestoData } = await import('./manifesto.js'); + const manifestoResult = manifestoData(dbPath, { + file: args.file, + noTests: args.no_tests, + kind: args.kind, + limit: Math.min(args.limit ?? MCP_DEFAULTS.manifesto, MCP_MAX_LIMIT), + offset: args.offset ?? 0, + }); + result = { check: checkResult, manifesto: manifestoResult }; + } else { + result = checkResult; + } + } break; } case 'ast_query': { diff --git a/tests/unit/mcp.test.js b/tests/unit/mcp.test.js index 68ab0718..4e5bc3cc 100644 --- a/tests/unit/mcp.test.js +++ b/tests/unit/mcp.test.js @@ -10,6 +10,7 @@ import { buildToolList, TOOLS } from '../../src/mcp.js'; const ALL_TOOL_NAMES = [ 'query', + 'path', 'file_deps', 'file_exports', 'impact_analysis', @@ -18,19 +19,16 @@ const ALL_TOOL_NAMES = [ 'fn_impact', 'context', 'symbol_children', - 'explain', 'where', 'diff_impact', 'semantic_search', 'export_graph', 'list_functions', 'structure', - 'hotspots', 'co_changes', 'node_roles', 'execution_flow', 'complexity', - 'manifesto', 'communities', 'code_owners', 'audit', @@ -75,6 +73,18 @@ describe('TOOLS', () => { expect(q.inputSchema.properties.kind.enum).toBeDefined(); }); + it('path requires from and to parameters', () => { + const p = TOOLS.find((t) => t.name === 'path'); + expect(p).toBeDefined(); + expect(p.inputSchema.required).toContain('from'); + expect(p.inputSchema.required).toContain('to'); + expect(p.inputSchema.properties).toHaveProperty('depth'); + expect(p.inputSchema.properties).toHaveProperty('edge_kinds'); + expect(p.inputSchema.properties).toHaveProperty('from_file'); + expect(p.inputSchema.properties).toHaveProperty('to_file'); + expect(p.inputSchema.properties).toHaveProperty('no_tests'); + }); + it('file_deps requires file parameter', () => { const fd = TOOLS.find((t) => t.name === 'file_deps'); expect(fd.inputSchema.required).toContain('file'); @@ -123,18 +133,23 @@ describe('TOOLS', () => { expect(di.inputSchema.properties).toHaveProperty('depth'); }); - it('check has no required parameters', () => { + it('check has no required parameters and includes manifesto props', () => { const ch = TOOLS.find((t) => t.name === 'check'); expect(ch).toBeDefined(); expect(ch.inputSchema.required).toBeUndefined(); expect(ch.inputSchema.properties).toHaveProperty('ref'); expect(ch.inputSchema.properties).toHaveProperty('staged'); + expect(ch.inputSchema.properties).toHaveProperty('rules'); expect(ch.inputSchema.properties).toHaveProperty('cycles'); expect(ch.inputSchema.properties).toHaveProperty('blast_radius'); expect(ch.inputSchema.properties).toHaveProperty('signatures'); expect(ch.inputSchema.properties).toHaveProperty('boundaries'); expect(ch.inputSchema.properties).toHaveProperty('depth'); + expect(ch.inputSchema.properties).toHaveProperty('file'); + expect(ch.inputSchema.properties).toHaveProperty('kind'); expect(ch.inputSchema.properties).toHaveProperty('no_tests'); + expect(ch.inputSchema.properties).toHaveProperty('limit'); + expect(ch.inputSchema.properties).toHaveProperty('offset'); }); it('semantic_search requires query parameter', () => { @@ -175,13 +190,24 @@ describe('TOOLS', () => { expect(st.inputSchema.properties).toHaveProperty('sort'); }); - it('hotspots has no required parameters', () => { - const hs = TOOLS.find((t) => t.name === 'hotspots'); - expect(hs).toBeDefined(); - expect(hs.inputSchema.required).toBeUndefined(); - expect(hs.inputSchema.properties).toHaveProperty('metric'); - expect(hs.inputSchema.properties).toHaveProperty('level'); - expect(hs.inputSchema.properties).toHaveProperty('limit'); + it('audit requires target and has quick param', () => { + const a = TOOLS.find((t) => t.name === 'audit'); + expect(a).toBeDefined(); + expect(a.inputSchema.required).toContain('target'); + expect(a.inputSchema.properties).toHaveProperty('quick'); + expect(a.inputSchema.properties).toHaveProperty('depth'); + expect(a.inputSchema.properties).toHaveProperty('limit'); + expect(a.inputSchema.properties).toHaveProperty('offset'); + }); + + it('triage has level param for hotspot routing', () => { + const tr = TOOLS.find((t) => t.name === 'triage'); + expect(tr).toBeDefined(); + expect(tr.inputSchema.required).toBeUndefined(); + expect(tr.inputSchema.properties).toHaveProperty('level'); + expect(tr.inputSchema.properties.level.enum).toEqual(['function', 'file', 'directory']); + expect(tr.inputSchema.properties).toHaveProperty('sort'); + expect(tr.inputSchema.properties).toHaveProperty('weights'); }); it('every tool except list_repos has optional repo property', () => { From a1583cb82f8ca0c61d81f8745ae4b0e39e3d9328 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Tue, 3 Mar 2026 02:02:50 -0700 Subject: [PATCH 4/4] fix: map triage sort values to valid hotspot metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the triage tool delegates to hotspotsData for file/directory level, sort values like 'complexity', 'churn', and 'mi' don't exist in HOTSPOT_QUERIES (which only supports fan-in, fan-out, density, coupling). Previously these silently fell back to fan-in via the || fallback. Add explicit TRIAGE_TO_HOTSPOT mapping: - risk → fan-in (unchanged) - complexity → density (symbol count as structural proxy) - churn → coupling (fan-in + fan-out reflects change propagation) - mi → fan-in (no direct equivalent) - fan-in → fan-in (pass-through via nullish coalescing) Impact: 1 functions changed, 1 affected --- src/mcp.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/mcp.js b/src/mcp.js index c1ad7a5c..eb8d1f0d 100644 --- a/src/mcp.js +++ b/src/mcp.js @@ -1234,7 +1234,13 @@ export async function startMCPServer(customDbPath, options = {}) { case 'triage': { if (args.level === 'file' || args.level === 'directory') { const { hotspotsData } = await import('./structure.js'); - const metric = args.sort === 'risk' ? 'fan-in' : args.sort; + const TRIAGE_TO_HOTSPOT = { + risk: 'fan-in', + complexity: 'density', + churn: 'coupling', + mi: 'fan-in', + }; + const metric = TRIAGE_TO_HOTSPOT[args.sort] ?? args.sort; result = hotspotsData(dbPath, { metric, level: args.level,