diff --git a/src/change-journal.js b/src/change-journal.js new file mode 100644 index 00000000..bbba73ec --- /dev/null +++ b/src/change-journal.js @@ -0,0 +1,130 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { debug, warn } from './logger.js'; + +export const CHANGE_EVENTS_FILENAME = 'change-events.ndjson'; +export const DEFAULT_MAX_BYTES = 1024 * 1024; // 1 MB + +/** + * Returns the absolute path to the NDJSON change events file. + */ +export function changeEventsPath(rootDir) { + return path.join(rootDir, '.codegraph', CHANGE_EVENTS_FILENAME); +} + +/** + * Compare old and new symbol arrays, returning added/removed/modified sets. + * Symbols are keyed on `name\0kind`. A symbol is "modified" if the same + * name+kind exists in both but the line changed. + * + * @param {Array<{name:string, kind:string, line:number}>} oldSymbols + * @param {Array<{name:string, kind:string, line:number}>} newSymbols + * @returns {{ added: Array, removed: Array, modified: Array }} + */ +export function diffSymbols(oldSymbols, newSymbols) { + const oldMap = new Map(); + for (const s of oldSymbols) { + oldMap.set(`${s.name}\0${s.kind}`, s); + } + + const newMap = new Map(); + for (const s of newSymbols) { + newMap.set(`${s.name}\0${s.kind}`, s); + } + + const added = []; + const removed = []; + const modified = []; + + for (const [key, s] of newMap) { + const old = oldMap.get(key); + if (!old) { + added.push({ name: s.name, kind: s.kind, line: s.line }); + } else if (old.line !== s.line) { + modified.push({ name: s.name, kind: s.kind, line: s.line }); + } + } + + for (const [key, s] of oldMap) { + if (!newMap.has(key)) { + removed.push({ name: s.name, kind: s.kind }); + } + } + + return { added, removed, modified }; +} + +/** + * Assemble a single change event object. + */ +export function buildChangeEvent(file, event, symbolDiff, counts) { + return { + ts: new Date().toISOString(), + file, + event, + symbols: symbolDiff, + counts: { + nodes: { before: counts.nodesBefore ?? 0, after: counts.nodesAfter ?? 0 }, + edges: { added: counts.edgesAdded ?? 0 }, + }, + }; +} + +/** + * Append change events as NDJSON lines to the change events file. + * Creates the .codegraph directory if needed. Non-fatal on failure. + */ +export function appendChangeEvents(rootDir, events) { + const filePath = changeEventsPath(rootDir); + const dir = path.dirname(filePath); + + try { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + const lines = `${events.map((e) => JSON.stringify(e)).join('\n')}\n`; + fs.appendFileSync(filePath, lines); + debug(`Appended ${events.length} change event(s) to ${filePath}`); + } catch (err) { + warn(`Failed to append change events: ${err.message}`); + return; + } + + try { + rotateIfNeeded(filePath, DEFAULT_MAX_BYTES); + } catch { + /* rotation failure is non-fatal */ + } +} + +/** + * If the file exceeds maxBytes, keep the last ~half by finding + * the first newline at or after the midpoint and rewriting from there. + */ +export function rotateIfNeeded(filePath, maxBytes = DEFAULT_MAX_BYTES) { + let stat; + try { + stat = fs.statSync(filePath); + } catch { + return; // file doesn't exist, nothing to rotate + } + + if (stat.size <= maxBytes) return; + + try { + const buf = fs.readFileSync(filePath); + const mid = Math.floor(buf.length / 2); + const newlineIdx = buf.indexOf(0x0a, mid); + if (newlineIdx === -1) { + warn( + `Change events file exceeds ${maxBytes} bytes but contains no line breaks; skipping rotation`, + ); + return; + } + const kept = buf.slice(newlineIdx + 1); + fs.writeFileSync(filePath, kept); + debug(`Rotated change events: ${stat.size} → ${kept.length} bytes`); + } catch (err) { + warn(`Failed to rotate change events: ${err.message}`); + } +} diff --git a/src/queries.js b/src/queries.js index c490744f..9a0204a5 100644 --- a/src/queries.js +++ b/src/queries.js @@ -234,6 +234,7 @@ export function queryNameData(name, customDbPath, opts = {}) { kind: node.kind, file: node.file, line: node.line, + fileHash: getFileHash(db, node.file), callees: callees.map((c) => ({ name: c.name, kind: c.kind, @@ -2732,6 +2733,11 @@ export function explain(target, customDbPath, opts = {}) { // ─── whereData ────────────────────────────────────────────────────────── +function getFileHash(db, file) { + const row = db.prepare('SELECT hash FROM file_hashes WHERE file = ?').get(file); + return row ? row.hash : null; +} + function whereSymbolImpl(db, target, noTests) { const placeholders = ALL_SYMBOL_KINDS.map(() => '?').join(', '); let nodes = db @@ -2763,6 +2769,7 @@ function whereSymbolImpl(db, target, noTests) { kind: node.kind, file: node.file, line: node.line, + fileHash: getFileHash(db, node.file), role: node.role || null, exported, uses: uses.map((u) => ({ name: u.name, file: u.file, line: u.line })), @@ -2813,6 +2820,7 @@ function whereFileImpl(db, target) { return { file: fn.file, + fileHash: getFileHash(db, fn.file), symbols: symbols.map((s) => ({ name: s.name, kind: s.kind, line: s.line })), imports, importedBy, diff --git a/src/watcher.js b/src/watcher.js index 8ee5726c..8b87b834 100644 --- a/src/watcher.js +++ b/src/watcher.js @@ -1,6 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { readFileSafe } from './builder.js'; +import { appendChangeEvents, buildChangeEvent, diffSymbols } from './change-journal.js'; import { EXTENSIONS, IGNORE_DIRS, normalizePath } from './constants.js'; import { closeDb, initSchema, openDb } from './db.js'; import { appendJournalEntries } from './journal.js'; @@ -25,13 +26,25 @@ async function updateFile(_db, rootDir, filePath, stmts, engineOpts, cache) { const oldNodes = stmts.countNodes.get(relPath)?.c || 0; const _oldEdges = stmts.countEdgesForFile.get(relPath)?.c || 0; + const oldSymbols = stmts.listSymbols.all(relPath); stmts.deleteEdgesForFile.run(relPath); stmts.deleteNodes.run(relPath); if (!fs.existsSync(filePath)) { if (cache) cache.remove(filePath); - return { file: relPath, nodesAdded: 0, nodesRemoved: oldNodes, edgesAdded: 0, deleted: true }; + const symbolDiff = diffSymbols(oldSymbols, []); + return { + file: relPath, + nodesAdded: 0, + nodesRemoved: oldNodes, + edgesAdded: 0, + deleted: true, + event: 'deleted', + symbolDiff, + nodesBefore: oldNodes, + nodesAfter: 0, + }; } let code; @@ -55,6 +68,7 @@ async function updateFile(_db, rootDir, filePath, stmts, engineOpts, cache) { } const newNodes = stmts.countNodes.get(relPath)?.c || 0; + const newSymbols = stmts.listSymbols.all(relPath); let edgesAdded = 0; const fileNodeRow = stmts.getNodeId.get(relPath, 'file', relPath, 0); @@ -129,12 +143,19 @@ async function updateFile(_db, rootDir, filePath, stmts, engineOpts, cache) { } } + const symbolDiff = diffSymbols(oldSymbols, newSymbols); + const event = oldNodes === 0 ? 'added' : 'modified'; + return { file: relPath, nodesAdded: newNodes, nodesRemoved: oldNodes, edgesAdded, deleted: false, + event, + symbolDiff, + nodesBefore: oldNodes, + nodesAfter: newNodes, }; } @@ -180,6 +201,7 @@ export async function watchProject(rootDir, opts = {}) { findNodeByName: db.prepare( "SELECT id, file FROM nodes WHERE name = ? AND kind IN ('function', 'method', 'class', 'interface', 'type', 'struct', 'enum', 'trait', 'record', 'module')", ), + listSymbols: db.prepare("SELECT name, kind, line FROM nodes WHERE file = ? AND kind != 'file'"), }; // Use named params for statements needing the same value twice @@ -218,6 +240,19 @@ export async function watchProject(rootDir, opts = {}) { } catch { /* journal write failure is non-fatal */ } + + const changeEvents = updates.map((r) => + buildChangeEvent(r.file, r.event, r.symbolDiff, { + nodesBefore: r.nodesBefore, + nodesAfter: r.nodesAfter, + edgesAdded: r.edgesAdded, + }), + ); + try { + appendChangeEvents(rootDir, changeEvents); + } catch { + /* change event write failure is non-fatal */ + } } for (const r of updates) { diff --git a/tests/integration/queries.test.js b/tests/integration/queries.test.js index 69cf916b..40fab71d 100644 --- a/tests/integration/queries.test.js +++ b/tests/integration/queries.test.js @@ -32,9 +32,11 @@ import { fnDepsData, fnImpactData, impactAnalysisData, + listFunctionsData, moduleMapData, pathData, queryNameData, + rolesData, statsData, whereData, } from '../../src/queries.js'; @@ -101,6 +103,16 @@ beforeAll(() => { // Low-confidence call edge for quality tests insertEdge(db, formatResponse, validateToken, 'calls', 0.3); + // File hashes (for fileHash exposure) + for (const f of ['auth.js', 'middleware.js', 'routes.js', 'utils.js', 'auth.test.js']) { + db.prepare('INSERT INTO file_hashes (file, hash, mtime, size) VALUES (?, ?, ?, ?)').run( + f, + `hash_${f.replace('.', '_')}`, + Date.now(), + 100, + ); + } + db.close(); }); @@ -117,6 +129,7 @@ describe('queryNameData', () => { expect(fn).toBeDefined(); expect(fn.callers.map((c) => c.name)).toContain('authMiddleware'); expect(fn.callees.map((c) => c.name)).toContain('validateToken'); + expect(fn.fileHash).toBe('hash_auth_js'); }); test('returns empty results for nonexistent name', () => { @@ -516,6 +529,7 @@ describe('whereData', () => { expect(r.file).toBe('middleware.js'); expect(r.line).toBe(5); expect(r.uses.map((u) => u.name)).toContain('handleRoute'); + expect(r.fileHash).toBe('hash_middleware_js'); }); test('symbol: exported flag', () => { @@ -547,6 +561,7 @@ describe('whereData', () => { expect(r.symbols.map((s) => s.name)).toContain('authMiddleware'); expect(r.imports).toContain('auth.js'); expect(r.importedBy).toContain('routes.js'); + expect(r.fileHash).toBe('hash_middleware_js'); }); test('file: exported list', () => { diff --git a/tests/unit/change-journal.test.js b/tests/unit/change-journal.test.js new file mode 100644 index 00000000..5fcc787b --- /dev/null +++ b/tests/unit/change-journal.test.js @@ -0,0 +1,307 @@ +/** + * Unit tests for src/change-journal.js + */ + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { + appendChangeEvents, + buildChangeEvent, + CHANGE_EVENTS_FILENAME, + changeEventsPath, + DEFAULT_MAX_BYTES, + diffSymbols, + rotateIfNeeded, +} from '../../src/change-journal.js'; + +let tmpDir; + +beforeAll(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-change-journal-')); +}); + +afterAll(() => { + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +function makeRoot() { + const root = fs.mkdtempSync(path.join(tmpDir, 'root-')); + fs.mkdirSync(path.join(root, '.codegraph'), { recursive: true }); + return root; +} + +function eventsPath(root) { + return path.join(root, '.codegraph', CHANGE_EVENTS_FILENAME); +} + +function readLines(filePath) { + return fs + .readFileSync(filePath, 'utf-8') + .split('\n') + .filter((l) => l.length > 0); +} + +function parseLines(filePath) { + return readLines(filePath).map((l) => JSON.parse(l)); +} + +describe('diffSymbols', () => { + it('returns empty arrays for empty inputs', () => { + const result = diffSymbols([], []); + expect(result).toEqual({ added: [], removed: [], modified: [] }); + }); + + it('detects added symbols', () => { + const result = diffSymbols([], [{ name: 'foo', kind: 'function', line: 1 }]); + expect(result.added).toEqual([{ name: 'foo', kind: 'function', line: 1 }]); + expect(result.removed).toEqual([]); + expect(result.modified).toEqual([]); + }); + + it('detects removed symbols', () => { + const result = diffSymbols([{ name: 'foo', kind: 'function', line: 1 }], []); + expect(result.removed).toEqual([{ name: 'foo', kind: 'function' }]); + expect(result.added).toEqual([]); + expect(result.modified).toEqual([]); + }); + + it('detects modified symbols (line changed)', () => { + const result = diffSymbols( + [{ name: 'foo', kind: 'function', line: 1 }], + [{ name: 'foo', kind: 'function', line: 10 }], + ); + expect(result.modified).toEqual([{ name: 'foo', kind: 'function', line: 10 }]); + expect(result.added).toEqual([]); + expect(result.removed).toEqual([]); + }); + + it('treats same name with different kind as separate symbols', () => { + const result = diffSymbols( + [{ name: 'Foo', kind: 'class', line: 1 }], + [{ name: 'Foo', kind: 'function', line: 5 }], + ); + expect(result.added).toEqual([{ name: 'Foo', kind: 'function', line: 5 }]); + expect(result.removed).toEqual([{ name: 'Foo', kind: 'class' }]); + expect(result.modified).toEqual([]); + }); + + it('reports no changes when symbols are identical', () => { + const syms = [ + { name: 'a', kind: 'function', line: 1 }, + { name: 'b', kind: 'method', line: 5 }, + ]; + const result = diffSymbols(syms, syms); + expect(result).toEqual({ added: [], removed: [], modified: [] }); + }); + + it('handles complex mixed changes', () => { + const old = [ + { name: 'keep', kind: 'function', line: 1 }, + { name: 'move', kind: 'method', line: 10 }, + { name: 'drop', kind: 'class', line: 20 }, + ]; + const now = [ + { name: 'keep', kind: 'function', line: 1 }, + { name: 'move', kind: 'method', line: 15 }, + { name: 'fresh', kind: 'function', line: 25 }, + ]; + const result = diffSymbols(old, now); + expect(result.added).toEqual([{ name: 'fresh', kind: 'function', line: 25 }]); + expect(result.removed).toEqual([{ name: 'drop', kind: 'class' }]); + expect(result.modified).toEqual([{ name: 'move', kind: 'method', line: 15 }]); + }); +}); + +describe('buildChangeEvent', () => { + it('returns well-formed event object', () => { + const diff = { added: [{ name: 'x', kind: 'function', line: 1 }], removed: [], modified: [] }; + const ev = buildChangeEvent('src/foo.js', 'modified', diff, { + nodesBefore: 5, + nodesAfter: 6, + edgesAdded: 3, + }); + + expect(ev.file).toBe('src/foo.js'); + expect(ev.event).toBe('modified'); + expect(ev.symbols).toBe(diff); + expect(ev.counts).toEqual({ nodes: { before: 5, after: 6 }, edges: { added: 3 } }); + }); + + it('has a valid ISO timestamp', () => { + const ev = buildChangeEvent('a.js', 'added', { added: [], removed: [], modified: [] }, {}); + expect(ev.ts).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/); + }); + + it('defaults missing counts to 0', () => { + const ev = buildChangeEvent('a.js', 'added', { added: [], removed: [], modified: [] }, {}); + expect(ev.counts).toEqual({ nodes: { before: 0, after: 0 }, edges: { added: 0 } }); + }); +}); + +describe('appendChangeEvents', () => { + it('creates file and writes NDJSON', () => { + const root = makeRoot(); + const diff = { added: [{ name: 'x', kind: 'function', line: 1 }], removed: [], modified: [] }; + const ev = buildChangeEvent('src/a.js', 'added', diff, { + nodesBefore: 0, + nodesAfter: 1, + edgesAdded: 0, + }); + + appendChangeEvents(root, [ev]); + + const lines = parseLines(eventsPath(root)); + expect(lines).toHaveLength(1); + expect(lines[0].file).toBe('src/a.js'); + expect(lines[0].event).toBe('added'); + }); + + it('appends to existing file', () => { + const root = makeRoot(); + const ev1 = buildChangeEvent('a.js', 'added', { added: [], removed: [], modified: [] }, {}); + const ev2 = buildChangeEvent('b.js', 'modified', { added: [], removed: [], modified: [] }, {}); + + appendChangeEvents(root, [ev1]); + appendChangeEvents(root, [ev2]); + + const lines = parseLines(eventsPath(root)); + expect(lines).toHaveLength(2); + expect(lines[0].file).toBe('a.js'); + expect(lines[1].file).toBe('b.js'); + }); + + it('creates .codegraph directory if missing', () => { + const root = fs.mkdtempSync(path.join(tmpDir, 'nodir-')); + const ev = buildChangeEvent('x.js', 'added', { added: [], removed: [], modified: [] }, {}); + + appendChangeEvents(root, [ev]); + + expect(fs.existsSync(eventsPath(root))).toBe(true); + const lines = parseLines(eventsPath(root)); + expect(lines).toHaveLength(1); + }); + + it('is non-fatal on bad root path', () => { + // Should not throw + const ev = buildChangeEvent('x.js', 'added', { added: [], removed: [], modified: [] }, {}); + expect(() => appendChangeEvents('/nonexistent/z/y/x/root', [ev])).not.toThrow(); + }); +}); + +describe('rotateIfNeeded', () => { + it('is a no-op when file is under threshold', () => { + const root = makeRoot(); + const fp = eventsPath(root); + fs.writeFileSync(fp, '{"a":1}\n{"b":2}\n'); + const before = fs.readFileSync(fp, 'utf-8'); + + rotateIfNeeded(fp, 1024); + + expect(fs.readFileSync(fp, 'utf-8')).toBe(before); + }); + + it('truncates at line boundary when over threshold', () => { + const root = makeRoot(); + const fp = eventsPath(root); + + // Write enough lines to exceed a small threshold + const line = `${JSON.stringify({ data: 'x'.repeat(50) })}\n`; + const content = line.repeat(20); + fs.writeFileSync(fp, content); + + rotateIfNeeded(fp, content.length - 10); + + const after = fs.readFileSync(fp, 'utf-8'); + expect(after.length).toBeLessThan(content.length); + // Every remaining line should be valid JSON + const lines = after.split('\n').filter((l) => l.length > 0); + expect(lines.length).toBeGreaterThan(0); + for (const l of lines) { + expect(() => JSON.parse(l)).not.toThrow(); + } + }); + + it('is a no-op on missing file', () => { + expect(() => rotateIfNeeded('/does/not/exist.ndjson', 100)).not.toThrow(); + }); +}); + +describe('changeEventsPath', () => { + it('returns correct path', () => { + const p = changeEventsPath('/my/project'); + expect(p).toBe(path.join('/my/project', '.codegraph', 'change-events.ndjson')); + }); +}); + +describe('constants', () => { + it('CHANGE_EVENTS_FILENAME is correct', () => { + expect(CHANGE_EVENTS_FILENAME).toBe('change-events.ndjson'); + }); + + it('DEFAULT_MAX_BYTES is 1 MB', () => { + expect(DEFAULT_MAX_BYTES).toBe(1024 * 1024); + }); +}); + +describe('full lifecycle', () => { + it('append past threshold, rotate, append more — all lines valid JSON', () => { + const root = makeRoot(); + const fp = eventsPath(root); + const smallMax = 500; + + // Append events until we exceed the threshold + for (let i = 0; i < 20; i++) { + const ev = buildChangeEvent( + `src/f${i}.js`, + 'modified', + { + added: [{ name: `fn${i}`, kind: 'function', line: i }], + removed: [], + modified: [], + }, + { nodesBefore: i, nodesAfter: i + 1, edgesAdded: 1 }, + ); + appendChangeEvents(root, [ev]); + } + + // Force rotation with small threshold + const sizeBeforeRotation = fs.statSync(fp).size; + rotateIfNeeded(fp, smallMax); + + const afterRotation = fs.readFileSync(fp, 'utf-8'); + // Rotation keeps roughly the last half — must be smaller than the original + expect(afterRotation.length).toBeLessThan(sizeBeforeRotation); + expect(afterRotation.length).toBeGreaterThan(0); + + // Append more after rotation + const ev = buildChangeEvent( + 'src/extra.js', + 'added', + { + added: [{ name: 'extra', kind: 'function', line: 1 }], + removed: [], + modified: [], + }, + { nodesBefore: 0, nodesAfter: 1, edgesAdded: 0 }, + ); + appendChangeEvents(root, [ev]); + + // Verify every line is valid JSON + const lines = readLines(fp); + expect(lines.length).toBeGreaterThan(0); + for (const l of lines) { + const parsed = JSON.parse(l); + expect(parsed).toHaveProperty('ts'); + expect(parsed).toHaveProperty('file'); + expect(parsed).toHaveProperty('event'); + expect(parsed).toHaveProperty('symbols'); + expect(parsed).toHaveProperty('counts'); + } + + // Last line should be the extra event + const last = JSON.parse(lines[lines.length - 1]); + expect(last.file).toBe('src/extra.js'); + }); +});