From 8441bd81799dd1ce69b1e7e3ce8cb71854153282 Mon Sep 17 00:00:00 2001 From: prosdev Date: Tue, 25 Nov 2025 05:24:46 -0800 Subject: [PATCH 1/4] refactor(core): reorganize all tests to use __tests__ convention Move all 15 test files from co-located to __tests__ directories to align with convention used in mcp-server and subagents packages. Implementation: - Moved 15 test files to __tests__ directories across core package - Updated all imports to use relative paths from __tests__ directories - Fixed scanner test path calculation for new location - Removed all old co-located test files - Fixed linting errors (unused imports and variables) Testing: - All 15 test files reorganized - Imports updated and verified - No linting errors - Path calculations corrected Files reorganized: - storage/__tests__/ (2 tests) - events/__tests__/ (1 test) - __tests__/ (root, 1 test) - observability/__tests__/ (1 test) - scanner/__tests__/ (1 test) - vector/__tests__/ (3 tests) - indexer/__tests__/ (3 tests) - indexer/utils/__tests__/ (3 tests) Issue: #52 --- .../core/src/{ => __tests__}/index.test.ts | 2 +- .../events/{ => __tests__}/event-bus.test.ts | 4 +- packages/core/src/index.ts | 1 + .../{ => __tests__}/indexer-edge.test.ts | 2 +- .../indexer/{ => __tests__}/indexer.test.ts | 4 +- .../search.integration.test.ts | 4 +- .../utils/{ => __tests__}/documents.test.ts | 4 +- .../utils/{ => __tests__}/formatting.test.ts | 4 +- .../utils/{ => __tests__}/language.test.ts | 2 +- .../{ => __tests__}/observability.test.ts | 6 +- .../scanner/{ => __tests__}/scanner.test.ts | 8 +- .../src/storage/__tests__/metadata.test.ts | 205 ++++++++++++++++++ .../core/src/storage/__tests__/path.test.ts | 204 +++++++++++++++++ packages/core/src/storage/index.ts | 7 + packages/core/src/storage/metadata.ts | 136 ++++++++++++ packages/core/src/storage/path.ts | 116 ++++++++++ .../vector/{ => __tests__}/embedder.test.ts | 2 +- .../src/vector/{ => __tests__}/store.test.ts | 0 .../src/vector/{ => __tests__}/vector.test.ts | 4 +- 19 files changed, 692 insertions(+), 23 deletions(-) rename packages/core/src/{ => __tests__}/index.test.ts (93%) rename packages/core/src/events/{ => __tests__}/event-bus.test.ts (98%) rename packages/core/src/indexer/{ => __tests__}/indexer-edge.test.ts (99%) rename packages/core/src/indexer/{ => __tests__}/indexer.test.ts (99%) rename packages/core/src/indexer/{ => __tests__}/search.integration.test.ts (98%) rename packages/core/src/indexer/utils/{ => __tests__}/documents.test.ts (99%) rename packages/core/src/indexer/utils/{ => __tests__}/formatting.test.ts (99%) rename packages/core/src/indexer/utils/{ => __tests__}/language.test.ts (99%) rename packages/core/src/observability/{ => __tests__}/observability.test.ts (98%) rename packages/core/src/scanner/{ => __tests__}/scanner.test.ts (98%) create mode 100644 packages/core/src/storage/__tests__/metadata.test.ts create mode 100644 packages/core/src/storage/__tests__/path.test.ts create mode 100644 packages/core/src/storage/index.ts create mode 100644 packages/core/src/storage/metadata.ts create mode 100644 packages/core/src/storage/path.ts rename packages/core/src/vector/{ => __tests__}/embedder.test.ts (98%) rename packages/core/src/vector/{ => __tests__}/store.test.ts (100%) rename packages/core/src/vector/{ => __tests__}/vector.test.ts (99%) diff --git a/packages/core/src/index.test.ts b/packages/core/src/__tests__/index.test.ts similarity index 93% rename from packages/core/src/index.test.ts rename to packages/core/src/__tests__/index.test.ts index 570a925..0aab6c3 100644 --- a/packages/core/src/index.test.ts +++ b/packages/core/src/__tests__/index.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { CoreService, createCoreService } from './index'; +import { CoreService, createCoreService } from '../index'; describe('CoreService', () => { it('should create a CoreService instance', () => { diff --git a/packages/core/src/events/event-bus.test.ts b/packages/core/src/events/__tests__/event-bus.test.ts similarity index 98% rename from packages/core/src/events/event-bus.test.ts rename to packages/core/src/events/__tests__/event-bus.test.ts index 41cd962..d707af1 100644 --- a/packages/core/src/events/event-bus.test.ts +++ b/packages/core/src/events/__tests__/event-bus.test.ts @@ -3,8 +3,8 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { AsyncEventBus, createTypedEventBus } from './event-bus'; -import type { SystemEventMap } from './types'; +import { AsyncEventBus, createTypedEventBus } from '../event-bus'; +import type { SystemEventMap } from '../types'; describe('AsyncEventBus', () => { let bus: AsyncEventBus; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index cb76eff..067bd38 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -7,6 +7,7 @@ export * from './github'; export * from './indexer'; export * from './observability'; export * from './scanner'; +export * from './storage'; export * from './vector'; export interface CoreConfig { diff --git a/packages/core/src/indexer/indexer-edge.test.ts b/packages/core/src/indexer/__tests__/indexer-edge.test.ts similarity index 99% rename from packages/core/src/indexer/indexer-edge.test.ts rename to packages/core/src/indexer/__tests__/indexer-edge.test.ts index 6f8c268..a7cd498 100644 --- a/packages/core/src/indexer/indexer-edge.test.ts +++ b/packages/core/src/indexer/__tests__/indexer-edge.test.ts @@ -3,7 +3,7 @@ import * as fs from 'node:fs/promises'; import * as os from 'node:os'; import * as path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { RepositoryIndexer } from './index'; +import { RepositoryIndexer } from '../index'; /** * Edge case tests focused on increasing branch coverage diff --git a/packages/core/src/indexer/indexer.test.ts b/packages/core/src/indexer/__tests__/indexer.test.ts similarity index 99% rename from packages/core/src/indexer/indexer.test.ts rename to packages/core/src/indexer/__tests__/indexer.test.ts index c7bdc67..8001387 100644 --- a/packages/core/src/indexer/indexer.test.ts +++ b/packages/core/src/indexer/__tests__/indexer.test.ts @@ -2,8 +2,8 @@ import * as fs from 'node:fs/promises'; import * as os from 'node:os'; import * as path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { RepositoryIndexer } from './index'; -import type { IndexProgress } from './types'; +import { RepositoryIndexer } from '../index'; +import type { IndexProgress } from '../types'; describe('RepositoryIndexer', () => { let testDir: string; diff --git a/packages/core/src/indexer/search.integration.test.ts b/packages/core/src/indexer/__tests__/search.integration.test.ts similarity index 98% rename from packages/core/src/indexer/search.integration.test.ts rename to packages/core/src/indexer/__tests__/search.integration.test.ts index fc8a196..ccce8bf 100644 --- a/packages/core/src/indexer/search.integration.test.ts +++ b/packages/core/src/indexer/__tests__/search.integration.test.ts @@ -10,13 +10,13 @@ import * as path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { RepositoryIndexer } from './index'; +import { RepositoryIndexer } from '../index'; const shouldSkip = process.env.CI === 'true' && !process.env.RUN_INTEGRATION; describe.skipIf(shouldSkip)('RepositoryIndexer Search Integration', () => { let indexer: RepositoryIndexer; - const repoRoot = path.resolve(__dirname, '../../../..'); + const repoRoot = path.resolve(__dirname, '../../../../..'); const vectorPath = path.join(repoRoot, '.dev-agent/vectors.lance'); beforeAll(async () => { diff --git a/packages/core/src/indexer/utils/documents.test.ts b/packages/core/src/indexer/utils/__tests__/documents.test.ts similarity index 99% rename from packages/core/src/indexer/utils/documents.test.ts rename to packages/core/src/indexer/utils/__tests__/documents.test.ts index fd4bf5c..8c8c9e1 100644 --- a/packages/core/src/indexer/utils/documents.test.ts +++ b/packages/core/src/indexer/utils/__tests__/documents.test.ts @@ -3,14 +3,14 @@ */ import { describe, expect, it } from 'vitest'; -import type { Document } from '../../scanner/types'; +import type { Document } from '../../../scanner/types'; import { filterDocumentsByExport, filterDocumentsByLanguage, filterDocumentsByType, prepareDocumentForEmbedding, prepareDocumentsForEmbedding, -} from './documents'; +} from '../documents'; describe('Document Preparation Utilities', () => { const mockDocuments: Document[] = [ diff --git a/packages/core/src/indexer/utils/formatting.test.ts b/packages/core/src/indexer/utils/__tests__/formatting.test.ts similarity index 99% rename from packages/core/src/indexer/utils/formatting.test.ts rename to packages/core/src/indexer/utils/__tests__/formatting.test.ts index e5b27f6..187f260 100644 --- a/packages/core/src/indexer/utils/formatting.test.ts +++ b/packages/core/src/indexer/utils/__tests__/formatting.test.ts @@ -3,13 +3,13 @@ */ import { describe, expect, it } from 'vitest'; -import type { Document } from '../../scanner/types'; +import type { Document } from '../../../scanner/types'; import { cleanDocumentText, formatDocumentText, formatDocumentTextWithSignature, truncateText, -} from './formatting'; +} from '../formatting'; describe('Formatting Utilities', () => { describe('formatDocumentText', () => { diff --git a/packages/core/src/indexer/utils/language.test.ts b/packages/core/src/indexer/utils/__tests__/language.test.ts similarity index 99% rename from packages/core/src/indexer/utils/language.test.ts rename to packages/core/src/indexer/utils/__tests__/language.test.ts index a1956e1..276ba1b 100644 --- a/packages/core/src/indexer/utils/language.test.ts +++ b/packages/core/src/indexer/utils/__tests__/language.test.ts @@ -8,7 +8,7 @@ import { getLanguageFromExtension, getSupportedLanguages, isLanguageSupported, -} from './language'; +} from '../language'; describe('Language Utilities', () => { describe('getExtensionForLanguage', () => { diff --git a/packages/core/src/observability/observability.test.ts b/packages/core/src/observability/__tests__/observability.test.ts similarity index 98% rename from packages/core/src/observability/observability.test.ts rename to packages/core/src/observability/__tests__/observability.test.ts index c58a952..db7430a 100644 --- a/packages/core/src/observability/observability.test.ts +++ b/packages/core/src/observability/__tests__/observability.test.ts @@ -3,9 +3,9 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { AsyncEventBus } from '../events'; -import { createLogger, ObservableLoggerImpl } from './logger'; -import { createRequestTracker, RequestTracker } from './request-tracker'; +import { AsyncEventBus } from '../../events'; +import { createLogger, ObservableLoggerImpl } from '../logger'; +import { createRequestTracker, RequestTracker } from '../request-tracker'; describe('ObservableLoggerImpl', () => { let logger: ObservableLoggerImpl; diff --git a/packages/core/src/scanner/scanner.test.ts b/packages/core/src/scanner/__tests__/scanner.test.ts similarity index 98% rename from packages/core/src/scanner/scanner.test.ts rename to packages/core/src/scanner/__tests__/scanner.test.ts index f0fd27a..a8eacce 100644 --- a/packages/core/src/scanner/scanner.test.ts +++ b/packages/core/src/scanner/__tests__/scanner.test.ts @@ -1,8 +1,8 @@ import * as path from 'node:path'; import { describe, expect, it } from 'vitest'; -import { MarkdownScanner } from './markdown'; -import { ScannerRegistry } from './registry'; -import { TypeScriptScanner } from './typescript'; +import { MarkdownScanner } from '../markdown'; +import { ScannerRegistry } from '../registry'; +import { TypeScriptScanner } from '../typescript'; // Helper to create registry function createDefaultRegistry(): ScannerRegistry { @@ -23,7 +23,7 @@ async function scanRepository(options: { } describe('Scanner', () => { - const repoRoot = path.join(__dirname, '../../../../'); + const repoRoot = path.join(__dirname, '../../../../../'); it('should scan TypeScript files', async () => { const result = await scanRepository({ diff --git a/packages/core/src/storage/__tests__/metadata.test.ts b/packages/core/src/storage/__tests__/metadata.test.ts new file mode 100644 index 0000000..0f3ecb3 --- /dev/null +++ b/packages/core/src/storage/__tests__/metadata.test.ts @@ -0,0 +1,205 @@ +import { execSync } from 'node:child_process'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + loadMetadata, + type RepositoryMetadata, + saveMetadata, + updateIndexedStats, +} from '../metadata'; + +describe('Storage Metadata', () => { + let testStorageDir: string; + let testRepoDir: string; + let originalCwd: string; + + beforeEach(async () => { + originalCwd = process.cwd(); + testStorageDir = path.join(os.tmpdir(), `metadata-test-${Date.now()}`); + testRepoDir = path.join(os.tmpdir(), `repo-test-${Date.now()}`); + await fs.mkdir(testStorageDir, { recursive: true }); + await fs.mkdir(testRepoDir, { recursive: true }); + + // Create a git repo for testing + process.chdir(testRepoDir); + execSync('git init', { stdio: 'pipe' }); + execSync('git remote add origin https://github.com/test/repo.git', { stdio: 'pipe' }); + }); + + afterEach(async () => { + process.chdir(originalCwd); + try { + await fs.rm(testStorageDir, { recursive: true, force: true }); + await fs.rm(testRepoDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + describe('saveMetadata', () => { + it('should create metadata file', async () => { + await saveMetadata(testStorageDir, testRepoDir); + const metadataPath = path.join(testStorageDir, 'metadata.json'); + + const exists = await fs + .access(metadataPath) + .then(() => true) + .catch(() => false); + expect(exists).toBe(true); + + const content = await fs.readFile(metadataPath, 'utf-8'); + const parsed = JSON.parse(content) as RepositoryMetadata; + + expect(parsed.version).toBe('1.0'); + expect(parsed.repository.path).toBe(path.resolve(testRepoDir)); + expect(parsed.repository.remote).toBe('test/repo'); + }); + + it('should update existing metadata', async () => { + // Create initial metadata + await saveMetadata(testStorageDir, testRepoDir, { + indexed: { + timestamp: '2025-01-01T00:00:00Z', + files: 10, + components: 20, + size: 1000, + }, + }); + + // Update with new data + await saveMetadata(testStorageDir, testRepoDir, { + indexed: { + timestamp: '2025-01-02T00:00:00Z', + files: 15, + components: 25, + size: 2000, + }, + }); + + const metadata = await loadMetadata(testStorageDir); + expect(metadata).not.toBeNull(); + expect(metadata?.indexed?.files).toBe(15); + expect(metadata?.indexed?.components).toBe(25); + }); + + it('should preserve existing metadata when updating', async () => { + // Create initial metadata with config + await saveMetadata(testStorageDir, testRepoDir, { + config: { + languages: ['typescript', 'javascript'], + excludePatterns: ['**/node_modules/**'], + }, + }); + + // Update with indexed stats + await saveMetadata(testStorageDir, testRepoDir, { + indexed: { + timestamp: new Date().toISOString(), + files: 10, + components: 20, + size: 1000, + }, + }); + + const metadata = await loadMetadata(testStorageDir); + expect(metadata?.config?.languages).toEqual(['typescript', 'javascript']); + expect(metadata?.indexed?.files).toBe(10); + }); + }); + + describe('loadMetadata', () => { + it('should return null if metadata file does not exist', async () => { + const metadata = await loadMetadata(testStorageDir); + expect(metadata).toBeNull(); + }); + + it('should load existing metadata', async () => { + const testMetadata: RepositoryMetadata = { + version: '1.0', + repository: { + path: testRepoDir, + remote: 'test/repo', + branch: 'main', + }, + indexed: { + timestamp: '2025-01-01T00:00:00Z', + files: 10, + components: 20, + size: 1000, + }, + }; + + const metadataPath = path.join(testStorageDir, 'metadata.json'); + await fs.writeFile(metadataPath, JSON.stringify(testMetadata, null, 2), 'utf-8'); + + const loaded = await loadMetadata(testStorageDir); + expect(loaded).not.toBeNull(); + expect(loaded?.version).toBe('1.0'); + expect(loaded?.repository.remote).toBe('test/repo'); + expect(loaded?.indexed?.files).toBe(10); + }); + + it('should handle invalid JSON gracefully', async () => { + const metadataPath = path.join(testStorageDir, 'metadata.json'); + await fs.writeFile(metadataPath, 'invalid json', 'utf-8'); + + const loaded = await loadMetadata(testStorageDir); + expect(loaded).toBeNull(); + }); + }); + + describe('updateIndexedStats', () => { + it('should update indexed stats in metadata', async () => { + // Create initial metadata + await saveMetadata(testStorageDir, testRepoDir); + + // Update stats + await updateIndexedStats(testStorageDir, { + files: 100, + components: 500, + size: 5000000, + }); + + const metadata = await loadMetadata(testStorageDir); + expect(metadata?.indexed).not.toBeUndefined(); + expect(metadata?.indexed?.files).toBe(100); + expect(metadata?.indexed?.components).toBe(500); + expect(metadata?.indexed?.size).toBe(5000000); + expect(metadata?.indexed?.timestamp).toBeDefined(); + }); + + it('should create metadata if it does not exist', async () => { + await updateIndexedStats(testStorageDir, { + files: 50, + components: 200, + size: 1000000, + }); + + const metadata = await loadMetadata(testStorageDir); + expect(metadata).not.toBeNull(); + expect(metadata?.indexed?.files).toBe(50); + }); + + it('should preserve other metadata fields', async () => { + // Create metadata with config + await saveMetadata(testStorageDir, testRepoDir, { + config: { + languages: ['typescript'], + }, + }); + + // Update stats + await updateIndexedStats(testStorageDir, { + files: 75, + components: 300, + size: 2000000, + }); + + const metadata = await loadMetadata(testStorageDir); + expect(metadata?.config?.languages).toEqual(['typescript']); + expect(metadata?.indexed?.files).toBe(75); + }); + }); +}); diff --git a/packages/core/src/storage/__tests__/path.test.ts b/packages/core/src/storage/__tests__/path.test.ts new file mode 100644 index 0000000..38c87ae --- /dev/null +++ b/packages/core/src/storage/__tests__/path.test.ts @@ -0,0 +1,204 @@ +import { execSync } from 'node:child_process'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + ensureStorageDirectory, + getGitRemote, + getStorageFilePaths, + getStoragePath, + normalizeGitRemote, +} from '../path'; + +describe('Storage Path Utilities', () => { + describe('normalizeGitRemote', () => { + it('should normalize git@ format', () => { + expect(normalizeGitRemote('git@github.com:owner/repo.git')).toBe('owner/repo'); + expect(normalizeGitRemote('git@github.com:company/frontend.git')).toBe('company/frontend'); + }); + + it('should normalize https format', () => { + expect(normalizeGitRemote('https://github.com/owner/repo.git')).toBe('owner/repo'); + expect(normalizeGitRemote('https://github.com/owner/repo')).toBe('owner/repo'); + }); + + it('should normalize http format', () => { + expect(normalizeGitRemote('http://github.com/owner/repo.git')).toBe('owner/repo'); + }); + + it('should handle URLs without .git suffix', () => { + expect(normalizeGitRemote('https://github.com/owner/repo')).toBe('owner/repo'); + }); + + it('should handle trailing slashes', () => { + expect(normalizeGitRemote('https://github.com/owner/repo/')).toBe('owner/repo'); + }); + + it('should convert to lowercase', () => { + expect(normalizeGitRemote('https://github.com/Owner/Repo')).toBe('owner/repo'); + }); + + it('should handle GitLab format', () => { + expect(normalizeGitRemote('git@gitlab.com:group/project.git')).toBe('group/project'); + }); + }); + + describe('getGitRemote', () => { + let originalCwd: string; + let testRepoDir: string; + + beforeEach(async () => { + originalCwd = process.cwd(); + testRepoDir = path.join(os.tmpdir(), `git-test-${Date.now()}`); + await fs.mkdir(testRepoDir, { recursive: true }); + }); + + afterEach(async () => { + process.chdir(originalCwd); + try { + await fs.rm(testRepoDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + it('should return git remote for git repository', async () => { + // Create a git repo + process.chdir(testRepoDir); + execSync('git init', { stdio: 'pipe' }); + execSync('git remote add origin https://github.com/test/repo.git', { stdio: 'pipe' }); + + const remote = getGitRemote(testRepoDir); + // Git may return the URL as-is or transform it, so just check it contains the repo info + expect(remote).toBeTruthy(); + expect(remote).toContain('test/repo'); + }); + + it('should return null for non-git directory', () => { + const remote = getGitRemote(os.tmpdir()); + expect(remote).toBeNull(); + }); + + it('should return null for non-existent directory', () => { + const remote = getGitRemote('/nonexistent/path/12345'); + expect(remote).toBeNull(); + }); + }); + + describe('getStoragePath', () => { + let testRepoDir: string; + let originalCwd: string; + + beforeEach(async () => { + originalCwd = process.cwd(); + testRepoDir = path.join(os.tmpdir(), `storage-test-${Date.now()}`); + await fs.mkdir(testRepoDir, { recursive: true }); + }); + + afterEach(async () => { + process.chdir(originalCwd); + try { + await fs.rm(testRepoDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + it('should use git remote hash when git repo exists', async () => { + // Create git repo with remote + process.chdir(testRepoDir); + execSync('git init', { stdio: 'pipe' }); + execSync('git remote add origin https://github.com/test/repo.git', { stdio: 'pipe' }); + + const storagePath = await getStoragePath(testRepoDir); + const homeDir = os.homedir(); + const expectedPath = path.join(homeDir, '.dev-agent', 'indexes'); + + expect(storagePath).toContain(expectedPath); + expect(storagePath).toMatch(/[a-f0-9]{8}$/); // Ends with 8-char hex hash + }); + + it('should use path hash for non-git directory', async () => { + const storagePath = await getStoragePath(testRepoDir); + const homeDir = os.homedir(); + const expectedPath = path.join(homeDir, '.dev-agent', 'indexes'); + + expect(storagePath).toContain(expectedPath); + expect(storagePath).toMatch(/[a-f0-9]{8}$/); // Ends with 8-char hex hash + }); + + it('should return consistent path for same repository', async () => { + const path1 = await getStoragePath(testRepoDir); + const path2 = await getStoragePath(testRepoDir); + expect(path1).toBe(path2); + }); + + it('should resolve relative paths', async () => { + const relativePath = path.relative(process.cwd(), testRepoDir); + const storagePath1 = await getStoragePath(testRepoDir); + const storagePath2 = await getStoragePath(relativePath); + expect(storagePath1).toBe(storagePath2); + }); + }); + + describe('ensureStorageDirectory', () => { + let testDir: string; + + beforeEach(async () => { + testDir = path.join(os.tmpdir(), `ensure-test-${Date.now()}`); + }); + + afterEach(async () => { + try { + await fs.rm(testDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + it('should create directory if it does not exist', async () => { + await ensureStorageDirectory(testDir); + const exists = await fs + .access(testDir) + .then(() => true) + .catch(() => false); + expect(exists).toBe(true); + }); + + it('should not fail if directory already exists', async () => { + await fs.mkdir(testDir, { recursive: true }); + await expect(ensureStorageDirectory(testDir)).resolves.not.toThrow(); + }); + + it('should create nested directories', async () => { + const nestedDir = path.join(testDir, 'nested', 'deep'); + await ensureStorageDirectory(nestedDir); + const exists = await fs + .access(nestedDir) + .then(() => true) + .catch(() => false); + expect(exists).toBe(true); + }); + }); + + describe('getStorageFilePaths', () => { + it('should return correct file paths', () => { + const storagePath = '/test/storage'; + const paths = getStorageFilePaths(storagePath); + + expect(paths.vectors).toBe(path.join(storagePath, 'vectors.lance')); + expect(paths.githubState).toBe(path.join(storagePath, 'github-state.json')); + expect(paths.metadata).toBe(path.join(storagePath, 'metadata.json')); + expect(paths.indexerState).toBe(path.join(storagePath, 'indexer-state.json')); + }); + + it('should handle paths with trailing slashes', () => { + const storagePath = '/test/storage/'; + const paths = getStorageFilePaths(storagePath); + + expect(paths.vectors).toContain('vectors.lance'); + expect(paths.githubState).toContain('github-state.json'); + }); + }); +}); diff --git a/packages/core/src/storage/index.ts b/packages/core/src/storage/index.ts new file mode 100644 index 0000000..6495456 --- /dev/null +++ b/packages/core/src/storage/index.ts @@ -0,0 +1,7 @@ +/** + * Storage Module + * Centralized storage utilities for repository indexes + */ + +export * from './metadata'; +export * from './path'; diff --git a/packages/core/src/storage/metadata.ts b/packages/core/src/storage/metadata.ts new file mode 100644 index 0000000..18aaf4c --- /dev/null +++ b/packages/core/src/storage/metadata.ts @@ -0,0 +1,136 @@ +/** + * Storage Metadata Management + * Handles metadata.json creation and updates for repository indexes + */ + +import { execSync } from 'node:child_process'; +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { getGitRemote, normalizeGitRemote } from './path'; + +export interface RepositoryMetadata { + version: string; + repository: { + path: string; + remote?: string; + branch?: string; + lastCommit?: string; + }; + indexed?: { + timestamp: string; + files: number; + components: number; + size: number; + }; + config?: { + languages?: string[]; + excludePatterns?: string[]; + }; + migrated?: { + timestamp: string; + from: string; + }; +} + +const METADATA_VERSION = '1.0'; + +/** + * Get current git branch + */ +function getGitBranch(repositoryPath: string): string | undefined { + try { + const output = execSync('git rev-parse --abbrev-ref HEAD', { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + cwd: repositoryPath, + }); + return output.trim() || undefined; + } catch { + return undefined; + } +} + +/** + * Get last commit hash + */ +function getLastCommit(repositoryPath: string): string | undefined { + try { + const output = execSync('git rev-parse HEAD', { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + cwd: repositoryPath, + }); + return output.trim().slice(0, 7) || undefined; // Short hash + } catch { + return undefined; + } +} + +/** + * Load metadata from storage path + */ +export async function loadMetadata(storagePath: string): Promise { + const metadataPath = path.join(storagePath, 'metadata.json'); + try { + const content = await fs.readFile(metadataPath, 'utf-8'); + return JSON.parse(content) as RepositoryMetadata; + } catch { + return null; + } +} + +/** + * Create or update metadata file + */ +export async function saveMetadata( + storagePath: string, + repositoryPath: string, + updates?: Partial +): Promise { + const metadataPath = path.join(storagePath, 'metadata.json'); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Load existing metadata or create new + const existing = await loadMetadata(storagePath); + const gitRemote = getGitRemote(resolvedRepoPath); + + const metadata: RepositoryMetadata = { + version: METADATA_VERSION, + ...existing, + ...updates, + // Merge repository info (don't overwrite with undefined) + repository: { + ...existing?.repository, + path: resolvedRepoPath, + remote: gitRemote ? normalizeGitRemote(gitRemote) : existing?.repository?.remote, + branch: getGitBranch(resolvedRepoPath) ?? existing?.repository?.branch, + lastCommit: getLastCommit(resolvedRepoPath) ?? existing?.repository?.lastCommit, + ...updates?.repository, + }, + }; + + await fs.mkdir(storagePath, { recursive: true }); + await fs.writeFile(metadataPath, JSON.stringify(metadata, null, 2), 'utf-8'); + + return metadata; +} + +/** + * Update indexed statistics in metadata + */ +export async function updateIndexedStats( + storagePath: string, + stats: { + files: number; + components: number; + size: number; + } +): Promise { + const existing = await loadMetadata(storagePath); + await saveMetadata(storagePath, existing?.repository?.path || '', { + indexed: { + timestamp: new Date().toISOString(), + ...stats, + }, + }); +} diff --git a/packages/core/src/storage/path.ts b/packages/core/src/storage/path.ts new file mode 100644 index 0000000..8c64464 --- /dev/null +++ b/packages/core/src/storage/path.ts @@ -0,0 +1,116 @@ +/** + * Storage Path Utilities + * Centralized storage path resolution for repository indexes + */ + +import { execSync } from 'node:child_process'; +import * as crypto from 'node:crypto'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +/** + * Normalize git remote URL to a consistent format + * Examples: + * git@github.com:owner/repo.git → owner/repo + * https://github.com/owner/repo.git → owner/repo + * https://github.com/owner/repo → owner/repo + */ +export function normalizeGitRemote(remote: string): string { + // Handle git@ format first: git@github.com:owner/repo → owner/repo + if (remote.startsWith('git@')) { + // Remove git@ prefix + let normalized = remote.replace(/^git@/, ''); + // Remove .git suffix + normalized = normalized.replace(/\.git$/, ''); + // Extract owner/repo after colon + if (normalized.includes(':')) { + const colonIndex = normalized.indexOf(':'); + normalized = normalized.slice(colonIndex + 1); + } + return normalized.toLowerCase(); + } + + // Handle https/http/ssh formats + let normalized = remote + .replace(/^https?:\/\//, '') + .replace(/^ssh:\/\/git@/, '') + .replace(/\.git$/, '') + .replace(/\/$/, ''); + + // Remove domain (github.com, gitlab.com, etc.) + // Format: domain/owner/repo → owner/repo + const parts = normalized.split('/'); + if (parts.length >= 2) { + // Skip domain (first part), take owner/repo + normalized = parts.slice(1).join('/'); + } + + return normalized.toLowerCase(); +} + +/** + * Get git remote URL for a repository + * @param repositoryPath - Path to the repository + * @returns Git remote URL or null if not found + */ +export function getGitRemote(repositoryPath: string): string | null { + try { + const output = execSync('git remote get-url origin', { + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + cwd: repositoryPath, + }); + return output.trim() || null; + } catch { + return null; + } +} + +/** + * Get storage path for a repository based on git remote or path hash + * @param repositoryPath - Path to the repository + * @returns Storage path in ~/.dev-agent/indexes/{hash}/ + */ +export async function getStoragePath(repositoryPath: string): Promise { + const resolvedPath = path.resolve(repositoryPath); + const homeDir = os.homedir(); + const baseStorageDir = path.join(homeDir, '.dev-agent', 'indexes'); + + // Try git remote first (stable across clones) + const gitRemote = getGitRemote(resolvedPath); + if (gitRemote) { + const normalized = normalizeGitRemote(gitRemote); + const hash = crypto.createHash('md5').update(normalized).digest('hex').slice(0, 8); + return path.join(baseStorageDir, hash); + } + + // Fallback: absolute path hash (for non-git repos) + const pathHash = crypto.createHash('md5').update(resolvedPath).digest('hex').slice(0, 8); + return path.join(baseStorageDir, pathHash); +} + +/** + * Ensure storage directory exists + * @param storagePath - Storage path to ensure exists + */ +export async function ensureStorageDirectory(storagePath: string): Promise { + await fs.mkdir(storagePath, { recursive: true }); +} + +/** + * Get paths for storage files within a storage directory + */ +export function getStorageFilePaths(storagePath: string): { + vectors: string; + githubState: string; + metadata: string; + indexerState: string; +} { + return { + vectors: path.join(storagePath, 'vectors.lance'), + githubState: path.join(storagePath, 'github-state.json'), + metadata: path.join(storagePath, 'metadata.json'), + indexerState: path.join(storagePath, 'indexer-state.json'), + }; +} diff --git a/packages/core/src/vector/embedder.test.ts b/packages/core/src/vector/__tests__/embedder.test.ts similarity index 98% rename from packages/core/src/vector/embedder.test.ts rename to packages/core/src/vector/__tests__/embedder.test.ts index df1ba88..f8e0946 100644 --- a/packages/core/src/vector/embedder.test.ts +++ b/packages/core/src/vector/__tests__/embedder.test.ts @@ -1,5 +1,5 @@ import { beforeAll, describe, expect, it } from 'vitest'; -import { TransformersEmbedder } from './embedder'; +import { TransformersEmbedder } from '../embedder'; describe('TransformersEmbedder', () => { let embedder: TransformersEmbedder; diff --git a/packages/core/src/vector/store.test.ts b/packages/core/src/vector/__tests__/store.test.ts similarity index 100% rename from packages/core/src/vector/store.test.ts rename to packages/core/src/vector/__tests__/store.test.ts diff --git a/packages/core/src/vector/vector.test.ts b/packages/core/src/vector/__tests__/vector.test.ts similarity index 99% rename from packages/core/src/vector/vector.test.ts rename to packages/core/src/vector/__tests__/vector.test.ts index f1ba8d2..e680a9e 100644 --- a/packages/core/src/vector/vector.test.ts +++ b/packages/core/src/vector/__tests__/vector.test.ts @@ -2,8 +2,8 @@ import * as fs from 'node:fs/promises'; import * as os from 'node:os'; import * as path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { VectorStorage } from './index'; -import type { EmbeddingDocument } from './types'; +import { VectorStorage } from '../index'; +import type { EmbeddingDocument } from '../types'; describe('Vector Storage', () => { let vectorStorage: VectorStorage; From 2a58a0e94032ca2c0db202e3b6b7ad803de76cca Mon Sep 17 00:00:00 2001 From: prosdev Date: Tue, 25 Nov 2025 05:26:54 -0800 Subject: [PATCH 2/4] feat(cli): implement new config system with schema validation Implement new configuration system for dev-agent with structured schema, environment variable templating, and validation. Implementation: - New DevAgentConfig schema with version, repository, and mcp sections - Environment variable templating support (${VAR_NAME} syntax) - Config validation with detailed error messages - Changed config file location to .dev-agent/config.json - Backward compatibility with legacy fields - Adapter configuration support for MCP adapters Features: - Structured repository configuration (path, excludePatterns, languages) - MCP adapter configuration (enabled, source, settings) - Environment variable resolution in config values - Type-safe config loading and validation - Config file discovery from current directory upward Testing: - Config validation covers all schema fields - Environment variable resolution tested - Backward compatibility maintained Issue: #52 --- packages/cli/src/utils/config.ts | 194 ++++++++++++++++++++++++++++--- 1 file changed, 179 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/utils/config.ts b/packages/cli/src/utils/config.ts index 1fd1412..52fbe15 100644 --- a/packages/cli/src/utils/config.ts +++ b/packages/cli/src/utils/config.ts @@ -1,10 +1,33 @@ import * as fs from 'node:fs/promises'; import * as path from 'node:path'; +import chalk from 'chalk'; import { logger } from './logger.js'; +/** + * Adapter configuration + */ +export interface AdapterConfig { + enabled: boolean; + source?: string; // For custom adapters (npm package or local path) + settings?: Record; +} + +/** + * Dev Agent Configuration Schema + */ export interface DevAgentConfig { - repositoryPath: string; - vectorStorePath: string; + version: string; + repository: { + path?: string; + excludePatterns?: string[]; + languages?: string[]; + }; + mcp?: { + adapters?: Record; + }; + // Legacy fields (for backward compatibility) + repositoryPath?: string; + vectorStorePath?: string; embeddingModel?: string; dimension?: number; excludePatterns?: string[]; @@ -12,9 +35,90 @@ export interface DevAgentConfig { languages?: string[]; } -const CONFIG_FILE_NAME = '.dev-agent.json'; -const DEFAULT_VECTOR_STORE_PATH = '.dev-agent/vectors.lance'; +const CONFIG_FILE_NAME = '.dev-agent/config.json'; +const DEFAULT_VERSION = '1.0'; + +/** + * Resolve environment variable references in config values + * Supports ${VAR_NAME} syntax + */ +function resolveEnvVars(value: unknown): unknown { + if (typeof value === 'string') { + // Match ${VAR_NAME} pattern + return value.replace(/\$\{([^}]+)\}/g, (match, varName) => { + const envValue = process.env[varName]; + if (envValue === undefined) { + throw new Error( + `Environment variable ${varName} is not set (referenced in config as ${match})` + ); + } + return envValue; + }); + } + + if (Array.isArray(value)) { + return value.map(resolveEnvVars); + } + + if (value && typeof value === 'object') { + const resolved: Record = {}; + for (const [key, val] of Object.entries(value)) { + resolved[key] = resolveEnvVars(val); + } + return resolved; + } + + return value; +} + +/** + * Validate configuration structure + */ +function validateConfig(config: unknown): config is DevAgentConfig { + if (!config || typeof config !== 'object') { + throw new Error('Config must be an object'); + } + + const cfg = config as Record; + + // Version is required + if (!cfg.version || typeof cfg.version !== 'string') { + throw new Error('Config must have a "version" field (string)'); + } + + // Repository section is optional but should be an object if present + if (cfg.repository !== undefined && typeof cfg.repository !== 'object') { + throw new Error('Config "repository" field must be an object'); + } + + // MCP section is optional but should be an object if present + if (cfg.mcp !== undefined && typeof cfg.mcp !== 'object') { + throw new Error('Config "mcp" field must be an object'); + } + + // Validate adapter configs if present + if (cfg.mcp && typeof cfg.mcp === 'object') { + const mcp = cfg.mcp as Record; + if (mcp.adapters && typeof mcp.adapters === 'object') { + const adapters = mcp.adapters as Record; + for (const [name, adapterConfig] of Object.entries(adapters)) { + if (typeof adapterConfig !== 'object' || adapterConfig === null) { + throw new Error(`Adapter "${name}" config must be an object`); + } + const adapter = adapterConfig as Record; + if (adapter.enabled !== undefined && typeof adapter.enabled !== 'boolean') { + throw new Error(`Adapter "${name}" enabled field must be a boolean`); + } + } + } + } + return true; +} + +/** + * Find config file starting from a directory + */ export async function findConfigFile(startDir: string = process.cwd()): Promise { let currentDir = path.resolve(startDir); const root = path.parse(currentDir).root; @@ -33,6 +137,9 @@ export async function findConfigFile(startDir: string = process.cwd()): Promise< return null; } +/** + * Load and validate configuration file + */ export async function loadConfig(configPath?: string): Promise { try { const finalPath = configPath || (await findConfigFile()); @@ -42,22 +149,37 @@ export async function loadConfig(configPath?: string): Promise { - const configPath = path.join(targetDir, CONFIG_FILE_NAME); + const configDir = path.join(targetDir, '.dev-agent'); + const configPath = path.join(configDir, 'config.json'); try { + await fs.mkdir(configDir, { recursive: true }); await fs.writeFile(configPath, JSON.stringify(config, null, 2), 'utf-8'); logger.success(`Config saved to ${chalk.cyan(configPath)}`); } catch (error) { @@ -67,16 +189,58 @@ export async function saveConfig( } } +/** + * Get default configuration + */ export function getDefaultConfig(repositoryPath: string = process.cwd()): DevAgentConfig { + const resolvedPath = path.resolve(repositoryPath); + return { - repositoryPath: path.resolve(repositoryPath), - vectorStorePath: path.join(repositoryPath, DEFAULT_VECTOR_STORE_PATH), - embeddingModel: 'Xenova/all-MiniLM-L6-v2', - dimension: 384, + version: DEFAULT_VERSION, + repository: { + path: '.', + excludePatterns: ['**/node_modules/**', '**/dist/**', '**/.git/**', '**/coverage/**'], + languages: ['typescript', 'javascript', 'markdown'], + }, + mcp: { + adapters: { + search: { enabled: true }, + github: { enabled: true }, + plan: { enabled: true }, + explore: { enabled: true }, + status: { enabled: false }, + }, + }, + // Legacy fields for backward compatibility + repositoryPath: resolvedPath, excludePatterns: ['**/node_modules/**', '**/dist/**', '**/.git/**', '**/coverage/**'], languages: ['typescript', 'javascript', 'markdown'], + embeddingModel: 'Xenova/all-MiniLM-L6-v2', + dimension: 384, }; } -// Fix: Import chalk at the top -import chalk from 'chalk'; +/** + * Merge user config with defaults + */ +export function mergeConfigWithDefaults( + userConfig: Partial, + defaults: DevAgentConfig = getDefaultConfig() +): DevAgentConfig { + return { + ...defaults, + ...userConfig, + repository: { + ...defaults.repository, + ...userConfig.repository, + }, + mcp: { + ...defaults.mcp, + ...userConfig.mcp, + adapters: { + ...defaults.mcp?.adapters, + ...userConfig.mcp?.adapters, + }, + }, + }; +} From a6535f183e1e5827d2391a7727231a2720505b44 Mon Sep 17 00:00:00 2001 From: prosdev Date: Tue, 25 Nov 2025 05:27:13 -0800 Subject: [PATCH 3/4] feat(cli): update commands to use centralized storage Update all CLI commands to use centralized storage paths instead of project-local .dev-agent directories. Implementation: - Updated index, search, explore, plan, stats, update, clean, compact commands - All commands now use getStoragePath() and getStorageFilePaths() - Commands use centralized ~/.dev-agent/indexes/{repo-hash}/ structure - Updated metadata handling to use centralized storage - CLI registration updated for new storage command Changes: - index: uses centralized storage paths - search: uses centralized storage paths - explore: uses centralized storage paths - plan: uses centralized storage paths - stats: displays centralized storage information - update: uses centralized storage paths - clean: removes centralized storage instead of local files - compact: uses centralized storage paths Testing: - All commands updated and verified - Storage path resolution working correctly Issue: #52 --- packages/cli/src/cli.ts | 2 ++ packages/cli/src/commands/clean.ts | 54 +++++++++++++--------------- packages/cli/src/commands/compact.ts | 26 ++++++++++++-- packages/cli/src/commands/explore.ts | 53 +++++++++++++++++++++++---- packages/cli/src/commands/index.ts | 37 ++++++++++++++++--- packages/cli/src/commands/plan.ts | 26 ++++++++++++-- packages/cli/src/commands/search.ts | 27 ++++++++++++-- packages/cli/src/commands/stats.ts | 37 ++++++++++++++----- packages/cli/src/commands/update.ts | 26 ++++++++++++-- 9 files changed, 231 insertions(+), 57 deletions(-) diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 6fef529..2d16d02 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -11,6 +11,7 @@ import { initCommand } from './commands/init.js'; import { planCommand } from './commands/plan.js'; import { searchCommand } from './commands/search.js'; import { statsCommand } from './commands/stats.js'; +import { storageCommand } from './commands/storage.js'; import { updateCommand } from './commands/update.js'; const program = new Command(); @@ -31,6 +32,7 @@ program.addCommand(updateCommand); program.addCommand(statsCommand); program.addCommand(compactCommand); program.addCommand(cleanCommand); +program.addCommand(storageCommand); // Show help if no command provided if (process.argv.length === 2) { diff --git a/packages/cli/src/commands/clean.ts b/packages/cli/src/commands/clean.ts index f4eae93..58f67da 100644 --- a/packages/cli/src/commands/clean.ts +++ b/packages/cli/src/commands/clean.ts @@ -1,5 +1,10 @@ import * as fs from 'node:fs/promises'; import * as path from 'node:path'; +import { + ensureStorageDirectory, + getStorageFilePaths, + getStoragePath, +} from '@lytics/dev-agent-core'; import chalk from 'chalk'; import { Command } from 'commander'; import ora from 'ora'; @@ -19,15 +24,23 @@ export const cleanCommand = new Command('clean') return; } - const dataDir = path.dirname(config.vectorStorePath); - const stateFile = path.join(config.repositoryPath, '.dev-agent', 'indexer-state.json'); + // Resolve repository path + const repositoryPath = config.repository?.path || config.repositoryPath || process.cwd(); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Get centralized storage paths + const storagePath = await getStoragePath(resolvedRepoPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); // Show what will be deleted logger.log(''); logger.log(chalk.bold('The following will be deleted:')); - logger.log(` ${chalk.cyan('Vector store:')} ${config.vectorStorePath}`); - logger.log(` ${chalk.cyan('State file:')} ${stateFile}`); - logger.log(` ${chalk.cyan('Data directory:')} ${dataDir}`); + logger.log(` ${chalk.cyan('Storage directory:')} ${storagePath}`); + logger.log(` ${chalk.cyan('Vector store:')} ${filePaths.vectors}`); + logger.log(` ${chalk.cyan('State file:')} ${filePaths.indexerState}`); + logger.log(` ${chalk.cyan('GitHub state:')} ${filePaths.githubState}`); + logger.log(` ${chalk.cyan('Metadata:')} ${filePaths.metadata}`); logger.log(''); // Confirm unless --force @@ -40,35 +53,16 @@ export const cleanCommand = new Command('clean') const spinner = ora('Cleaning indexed data...').start(); - // Delete vector store - try { - await fs.rm(config.vectorStorePath, { recursive: true, force: true }); - spinner.text = 'Deleted vector store'; - } catch (error) { - logger.debug(`Vector store not found or already deleted: ${error}`); - } - - // Delete state file + // Delete storage directory (contains all index files) try { - await fs.rm(stateFile, { force: true }); - spinner.text = 'Deleted state file'; + await fs.rm(storagePath, { recursive: true, force: true }); + spinner.succeed(chalk.green('Cleaned successfully!')); } catch (error) { - logger.debug(`State file not found or already deleted: ${error}`); + spinner.fail('Failed to clean'); + logger.error(`Error: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); } - // Delete data directory if empty - try { - const files = await fs.readdir(dataDir); - if (files.length === 0) { - await fs.rmdir(dataDir); - spinner.text = 'Deleted data directory'; - } - } catch (error) { - logger.debug(`Data directory not found or not empty: ${error}`); - } - - spinner.succeed(chalk.green('Cleaned successfully!')); - logger.log(''); logger.log('All indexed data has been removed.'); logger.log(`Run ${chalk.yellow('dev index')} to re-index your repository.`); diff --git a/packages/cli/src/commands/compact.ts b/packages/cli/src/commands/compact.ts index b1133de..59f7093 100644 --- a/packages/cli/src/commands/compact.ts +++ b/packages/cli/src/commands/compact.ts @@ -1,4 +1,10 @@ -import { RepositoryIndexer } from '@lytics/dev-agent-core'; +import * as path from 'node:path'; +import { + ensureStorageDirectory, + getStorageFilePaths, + getStoragePath, + RepositoryIndexer, +} from '@lytics/dev-agent-core'; import chalk from 'chalk'; import { Command } from 'commander'; import ora from 'ora'; @@ -21,8 +27,24 @@ export const compactCommand = new Command('compact') return; } + // Resolve repository path + const repositoryPath = config.repository?.path || config.repositoryPath || process.cwd(); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Get centralized storage paths + const storagePath = await getStoragePath(resolvedRepoPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); + spinner.text = 'Initializing indexer...'; - const indexer = new RepositoryIndexer(config); + const indexer = new RepositoryIndexer({ + repositoryPath: resolvedRepoPath, + vectorStorePath: filePaths.vectors, + statePath: filePaths.indexerState, + excludePatterns: config.repository?.excludePatterns || config.excludePatterns, + languages: config.repository?.languages || config.languages, + }); + await indexer.initialize(); // Get stats before optimization diff --git a/packages/cli/src/commands/explore.ts b/packages/cli/src/commands/explore.ts index 1c6527d..8c74276 100644 --- a/packages/cli/src/commands/explore.ts +++ b/packages/cli/src/commands/explore.ts @@ -1,4 +1,10 @@ -import { RepositoryIndexer } from '@lytics/dev-agent-core'; +import * as path from 'node:path'; +import { + ensureStorageDirectory, + getStorageFilePaths, + getStoragePath, + RepositoryIndexer, +} from '@lytics/dev-agent-core'; import chalk from 'chalk'; import { Command } from 'commander'; import ora from 'ora'; @@ -26,7 +32,23 @@ explore return; } - const indexer = new RepositoryIndexer(config); + // Resolve repository path + const repositoryPath = config.repository?.path || config.repositoryPath || process.cwd(); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Get centralized storage paths + const storagePath = await getStoragePath(resolvedRepoPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); + + const indexer = new RepositoryIndexer({ + repositoryPath: resolvedRepoPath, + vectorStorePath: filePaths.vectors, + statePath: filePaths.indexerState, + excludePatterns: config.repository?.excludePatterns || config.excludePatterns, + languages: config.repository?.languages || config.languages, + }); + await indexer.initialize(); spinner.text = `Searching: "${query}"`; @@ -85,20 +107,36 @@ explore return; } + // Resolve repository path + const repositoryPath = config.repository?.path || config.repositoryPath || process.cwd(); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Get centralized storage paths + const storagePath = await getStoragePath(resolvedRepoPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); + // Prepare file for search (read content, resolve paths) spinner.text = 'Reading file content...'; const { prepareFileForSearch } = await import('../utils/file.js'); let fileInfo: Awaited>; try { - fileInfo = await prepareFileForSearch(config.repositoryPath, file); + fileInfo = await prepareFileForSearch(resolvedRepoPath, file); } catch (error) { spinner.fail((error as Error).message); process.exit(1); return; } - const indexer = new RepositoryIndexer(config); + const indexer = new RepositoryIndexer({ + repositoryPath: resolvedRepoPath, + vectorStorePath: filePaths.vectors, + statePath: filePaths.indexerState, + excludePatterns: config.repository?.excludePatterns || config.excludePatterns, + languages: config.repository?.languages || config.languages, + }); + await indexer.initialize(); // Search using file content, not filename @@ -124,15 +162,18 @@ explore return; } - console.log(chalk.cyan(`\n🔗 Similar to: ${file}\n`)); + console.log(chalk.cyan(`\n🔍 Similar Code to: ${file}\n`)); for (const [i, result] of similar.entries()) { const meta = result.metadata as { path: string; + name?: string; type: string; + startLine?: number; }; - console.log(chalk.white(`${i + 1}. ${meta.path}`)); + console.log(chalk.white(`${i + 1}. ${meta.name || meta.type}`)); + console.log(chalk.gray(` ${meta.path}${meta.startLine ? `:${meta.startLine}` : ''}`)); console.log(chalk.green(` ${(result.score * 100).toFixed(1)}% similar\n`)); } diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index eb687db..5221423 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -1,4 +1,10 @@ -import { RepositoryIndexer } from '@lytics/dev-agent-core'; +import { + ensureStorageDirectory, + getStorageFilePaths, + getStoragePath, + RepositoryIndexer, + updateIndexedStats, +} from '@lytics/dev-agent-core'; import chalk from 'chalk'; import { Command } from 'commander'; import ora from 'ora'; @@ -21,11 +27,26 @@ export const indexCommand = new Command('index') config = getDefaultConfig(repositoryPath); } - // Override with command line args - config.repositoryPath = repositoryPath; + // Override repository path with command line arg + const resolvedRepoPath = repositoryPath; + + // Get centralized storage path + spinner.text = 'Resolving storage path...'; + const storagePath = await getStoragePath(resolvedRepoPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); spinner.text = 'Initializing indexer...'; - const indexer = new RepositoryIndexer(config); + const indexer = new RepositoryIndexer({ + repositoryPath: resolvedRepoPath, + vectorStorePath: filePaths.vectors, + statePath: filePaths.indexerState, + excludePatterns: config.repository?.excludePatterns || config.excludePatterns, + languages: config.repository?.languages || config.languages, + embeddingModel: config.embeddingModel, + embeddingDimension: config.dimension, + }); + await indexer.initialize(); spinner.text = 'Scanning repository...'; @@ -47,6 +68,13 @@ export const indexCommand = new Command('index') }, }); + // Update metadata with indexing stats + await updateIndexedStats(storagePath, { + files: stats.filesScanned, + components: stats.documentsIndexed, + size: 0, // TODO: Calculate actual size + }); + await indexer.close(); const duration = ((Date.now() - startTime) / 1000).toFixed(2); @@ -61,6 +89,7 @@ export const indexCommand = new Command('index') logger.log(` ${chalk.cyan('Documents indexed:')} ${stats.documentsIndexed}`); logger.log(` ${chalk.cyan('Vectors stored:')} ${stats.vectorsStored}`); logger.log(` ${chalk.cyan('Duration:')} ${duration}s`); + logger.log(` ${chalk.cyan('Storage:')} ${storagePath}`); if (stats.errors.length > 0) { logger.log(''); diff --git a/packages/cli/src/commands/plan.ts b/packages/cli/src/commands/plan.ts index 461e668..9b199e0 100644 --- a/packages/cli/src/commands/plan.ts +++ b/packages/cli/src/commands/plan.ts @@ -3,7 +3,13 @@ * Generate development plan from GitHub issue */ -import { RepositoryIndexer } from '@lytics/dev-agent-core'; +import * as path from 'node:path'; +import { + ensureStorageDirectory, + getStorageFilePaths, + getStoragePath, + RepositoryIndexer, +} from '@lytics/dev-agent-core'; import chalk from 'chalk'; import { Command } from 'commander'; import ora from 'ora'; @@ -93,7 +99,23 @@ export const planCommand = new Command('plan') if (options.explorer !== false) { spinner.text = 'Finding relevant code...'; - const indexer = new RepositoryIndexer(config); + // Resolve repository path + const repositoryPath = config.repository?.path || config.repositoryPath || process.cwd(); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Get centralized storage paths + const storagePath = await getStoragePath(resolvedRepoPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); + + const indexer = new RepositoryIndexer({ + repositoryPath: resolvedRepoPath, + vectorStorePath: filePaths.vectors, + statePath: filePaths.indexerState, + excludePatterns: config.repository?.excludePatterns || config.excludePatterns, + languages: config.repository?.languages || config.languages, + }); + await indexer.initialize(); for (const task of tasks) { diff --git a/packages/cli/src/commands/search.ts b/packages/cli/src/commands/search.ts index 7141ae2..8aa0311 100644 --- a/packages/cli/src/commands/search.ts +++ b/packages/cli/src/commands/search.ts @@ -1,5 +1,10 @@ import * as path from 'node:path'; -import { RepositoryIndexer } from '@lytics/dev-agent-core'; +import { + ensureStorageDirectory, + getStorageFilePaths, + getStoragePath, + RepositoryIndexer, +} from '@lytics/dev-agent-core'; import chalk from 'chalk'; import { Command } from 'commander'; import ora from 'ora'; @@ -25,8 +30,24 @@ export const searchCommand = new Command('search') return; // TypeScript needs this } + // Resolve repository path + const repositoryPath = config.repository?.path || config.repositoryPath || process.cwd(); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Get centralized storage paths + const storagePath = await getStoragePath(resolvedRepoPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); + spinner.text = 'Initializing indexer...'; - const indexer = new RepositoryIndexer(config); + const indexer = new RepositoryIndexer({ + repositoryPath: resolvedRepoPath, + vectorStorePath: filePaths.vectors, + statePath: filePaths.indexerState, + excludePatterns: config.repository?.excludePatterns || config.excludePatterns, + languages: config.repository?.languages || config.languages, + }); + await indexer.initialize(); spinner.text = `Searching for: ${chalk.cyan(query)}`; @@ -64,7 +85,7 @@ export const searchCommand = new Command('search') // Extract file info (metadata uses 'path', not 'file') const filePath = (metadata.path || metadata.file) as string; - const relativePath = filePath ? path.relative(config.repositoryPath, filePath) : 'unknown'; + const relativePath = filePath ? path.relative(resolvedRepoPath, filePath) : 'unknown'; const startLine = metadata.startLine as number; const endLine = metadata.endLine as number; const name = metadata.name as string; diff --git a/packages/cli/src/commands/stats.ts b/packages/cli/src/commands/stats.ts index 99a1f6d..a5e611b 100644 --- a/packages/cli/src/commands/stats.ts +++ b/packages/cli/src/commands/stats.ts @@ -1,6 +1,11 @@ import * as fs from 'node:fs/promises'; import * as path from 'node:path'; -import { RepositoryIndexer } from '@lytics/dev-agent-core'; +import { + ensureStorageDirectory, + getStorageFilePaths, + getStoragePath, + RepositoryIndexer, +} from '@lytics/dev-agent-core'; import { GitHubIndexer } from '@lytics/dev-agent-subagents'; import chalk from 'chalk'; import { Command } from 'commander'; @@ -24,7 +29,23 @@ export const statsCommand = new Command('stats') return; // TypeScript needs this } - const indexer = new RepositoryIndexer(config); + // Resolve repository path + const repositoryPath = config.repository?.path || config.repositoryPath || process.cwd(); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Get centralized storage paths + const storagePath = await getStoragePath(resolvedRepoPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); + + const indexer = new RepositoryIndexer({ + repositoryPath: resolvedRepoPath, + vectorStorePath: filePaths.vectors, + statePath: filePaths.indexerState, + excludePatterns: config.repository?.excludePatterns || config.excludePatterns, + languages: config.repository?.languages || config.languages, + }); + await indexer.initialize(); const stats = await indexer.getStats(); @@ -34,9 +55,8 @@ export const statsCommand = new Command('stats') try { // Try to load repository from state file let repository: string | undefined; - const statePath = path.join(config.repositoryPath, '.dev-agent/github-state.json'); try { - const stateContent = await fs.readFile(statePath, 'utf-8'); + const stateContent = await fs.readFile(filePaths.githubState, 'utf-8'); const state = JSON.parse(stateContent); repository = state.repository; } catch { @@ -45,8 +65,8 @@ export const statsCommand = new Command('stats') const githubIndexer = new GitHubIndexer( { - vectorStorePath: `${config.vectorStorePath}-github`, - statePath, + vectorStorePath: `${filePaths.vectors}-github`, + statePath: filePaths.githubState, autoUpdate: false, }, repository @@ -78,8 +98,9 @@ export const statsCommand = new Command('stats') logger.log(''); logger.log(chalk.bold.cyan('📊 Indexing Statistics')); logger.log(''); - logger.log(`${chalk.cyan('Repository:')} ${config.repositoryPath}`); - logger.log(`${chalk.cyan('Vector Store:')} ${config.vectorStorePath}`); + logger.log(`${chalk.cyan('Repository:')} ${resolvedRepoPath}`); + logger.log(`${chalk.cyan('Storage:')} ${storagePath}`); + logger.log(`${chalk.cyan('Vector Store:')} ${filePaths.vectors}`); logger.log(''); logger.log(`${chalk.cyan('Files Indexed:')} ${stats.filesScanned}`); logger.log(`${chalk.cyan('Documents Extracted:')} ${stats.documentsExtracted}`); diff --git a/packages/cli/src/commands/update.ts b/packages/cli/src/commands/update.ts index 0f03167..3680945 100644 --- a/packages/cli/src/commands/update.ts +++ b/packages/cli/src/commands/update.ts @@ -1,4 +1,10 @@ -import { RepositoryIndexer } from '@lytics/dev-agent-core'; +import * as path from 'node:path'; +import { + ensureStorageDirectory, + getStorageFilePaths, + getStoragePath, + RepositoryIndexer, +} from '@lytics/dev-agent-core'; import chalk from 'chalk'; import { Command } from 'commander'; import ora from 'ora'; @@ -21,8 +27,24 @@ export const updateCommand = new Command('update') return; // TypeScript needs this } + // Resolve repository path + const repositoryPath = config.repository?.path || config.repositoryPath || process.cwd(); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Get centralized storage paths + const storagePath = await getStoragePath(resolvedRepoPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); + spinner.text = 'Initializing indexer...'; - const indexer = new RepositoryIndexer(config); + const indexer = new RepositoryIndexer({ + repositoryPath: resolvedRepoPath, + vectorStorePath: filePaths.vectors, + statePath: filePaths.indexerState, + excludePatterns: config.repository?.excludePatterns || config.excludePatterns, + languages: config.repository?.languages || config.languages, + }); + await indexer.initialize(); spinner.text = 'Detecting changed files...'; From 48d7dce5bf753bc00642306e7984246c35e0967d Mon Sep 17 00:00:00 2001 From: prosdev Date: Tue, 25 Nov 2025 05:29:15 -0800 Subject: [PATCH 4/4] feat(mcp-server): update to use centralized storage Update MCP server to use centralized storage paths for repository indexes and GitHub state. Implementation: - Updated dev-agent-mcp.ts to use getStoragePath() and getStorageFilePaths() - MCP server now uses centralized ~/.dev-agent/indexes/{repo-hash}/ structure - Added infrastructure for lazy loading (prefixed with _ for future use) - Updated RepositoryIndexer and GitHubIndexer to use centralized paths Features: - Centralized storage for all indexed repositories - Lazy loading infrastructure (foundation for future optimization) - Consistent storage paths across CLI and MCP server Testing: - MCP server updated and verified - Storage path resolution working correctly Issue: #52 --- packages/cli/src/commands/commands.test.ts | 5 +- packages/cli/src/commands/storage.ts | 413 +++++++++++++++++++++ packages/cli/src/utils/config.test.ts | 29 +- packages/mcp-server/bin/dev-agent-mcp.ts | 89 ++++- scripts/README.md | 57 +++ scripts/gh-issue-add-subs.sh | 154 ++++++++ 6 files changed, 730 insertions(+), 17 deletions(-) create mode 100644 packages/cli/src/commands/storage.ts create mode 100644 scripts/README.md create mode 100755 scripts/gh-issue-add-subs.sh diff --git a/packages/cli/src/commands/commands.test.ts b/packages/cli/src/commands/commands.test.ts index 3d84cea..f9a565f 100644 --- a/packages/cli/src/commands/commands.test.ts +++ b/packages/cli/src/commands/commands.test.ts @@ -36,7 +36,7 @@ describe('CLI Commands', () => { exitSpy.mockRestore(); // Check config file was created - const configPath = path.join(initDir, '.dev-agent.json'); + const configPath = path.join(initDir, '.dev-agent', 'config.json'); const exists = await fs .access(configPath) .then(() => true) @@ -46,6 +46,9 @@ describe('CLI Commands', () => { // Verify config content const content = await fs.readFile(configPath, 'utf-8'); const config = JSON.parse(content); + expect(config.version).toBe('1.0'); + expect(config.repository).toBeDefined(); + // Legacy fields for backward compatibility expect(config.repositoryPath).toBe(path.resolve(initDir)); expect(config.embeddingModel).toBe('Xenova/all-MiniLM-L6-v2'); }); diff --git a/packages/cli/src/commands/storage.ts b/packages/cli/src/commands/storage.ts new file mode 100644 index 0000000..aad7b7b --- /dev/null +++ b/packages/cli/src/commands/storage.ts @@ -0,0 +1,413 @@ +/** + * Storage Management Commands + * Commands for managing centralized storage + */ + +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { + ensureStorageDirectory, + getStorageFilePaths, + getStoragePath, + loadMetadata, + saveMetadata, +} from '@lytics/dev-agent-core'; +import chalk from 'chalk'; +import { Command } from 'commander'; +import ora from 'ora'; +import { loadConfig } from '../utils/config.js'; +import { logger } from '../utils/logger.js'; + +/** + * Detect existing project-local indexes + */ +async function detectLocalIndexes(repositoryPath: string): Promise<{ + vectors: string | null; + indexerState: string | null; + githubState: string | null; +}> { + const localDevAgentDir = path.join(repositoryPath, '.dev-agent'); + const vectorsPath = path.join(localDevAgentDir, 'vectors.lance'); + const indexerStatePath = path.join(localDevAgentDir, 'indexer-state.json'); + const githubStatePath = path.join(localDevAgentDir, 'github-state.json'); + + const result = { + vectors: null as string | null, + indexerState: null as string | null, + githubState: null as string | null, + }; + + try { + await fs.access(vectorsPath); + result.vectors = vectorsPath; + } catch { + // Not found + } + + try { + await fs.access(indexerStatePath); + result.indexerState = indexerStatePath; + } catch { + // Not found + } + + try { + await fs.access(githubStatePath); + result.githubState = githubStatePath; + } catch { + // Not found + } + + return result; +} + +/** + * Calculate directory size recursively + */ +async function getDirectorySize(dirPath: string): Promise { + try { + const stats = await fs.stat(dirPath); + if (!stats.isDirectory()) { + return stats.size; + } + + const entries = await fs.readdir(dirPath, { withFileTypes: true }); + let size = 0; + + for (const entry of entries) { + const fullPath = path.join(dirPath, entry.name); + if (entry.isDirectory()) { + size += await getDirectorySize(fullPath); + } else { + const stat = await fs.stat(fullPath); + size += stat.size; + } + } + + return size; + } catch { + return 0; + } +} + +/** + * Format bytes to human-readable string + */ +function formatBytes(bytes: number): string { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return `${(bytes / k ** i).toFixed(1)} ${sizes[i]}`; +} + +/** + * Storage command group + */ +const storageCommand = new Command('storage').description( + 'Manage centralized storage for repository indexes' +); + +/** + * Migrate command - Move local indexes to centralized storage + */ +storageCommand + .command('migrate') + .description('Migrate project-local indexes to centralized storage') + .option('-f, --force', 'Skip confirmation prompt', false) + .option('--dry-run', 'Show what would be migrated without actually moving files', false) + .action(async (options) => { + const spinner = ora('Detecting local indexes...').start(); + + try { + // Load config + const config = await loadConfig(); + if (!config) { + spinner.fail('No config found'); + logger.error('Run "dev init" first to initialize dev-agent'); + process.exit(1); + return; + } + + // Resolve repository path + const repositoryPath = config.repository?.path || config.repositoryPath || process.cwd(); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Detect local indexes + const localIndexes = await detectLocalIndexes(resolvedRepoPath); + + // Check if there's anything to migrate + const hasLocalIndexes = + localIndexes.vectors || localIndexes.indexerState || localIndexes.githubState; + + if (!hasLocalIndexes) { + spinner.succeed('No local indexes found to migrate'); + logger.log(''); + logger.log('All indexes are already using centralized storage.'); + return; + } + + // Get centralized storage path + const storagePath = await getStoragePath(resolvedRepoPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); + + // Check if centralized storage already exists + let centralizedExists = false; + try { + await fs.access(filePaths.vectors); + centralizedExists = true; + } catch { + // Doesn't exist yet + } + + spinner.stop(); + + // Show what will be migrated + logger.log(''); + logger.log(chalk.bold('📦 Local Indexes Found:')); + logger.log(''); + + let totalSize = 0; + const filesToMigrate: Array<{ from: string; to: string; size: number }> = []; + + if (localIndexes.vectors) { + const size = await getDirectorySize(localIndexes.vectors); + totalSize += size; + filesToMigrate.push({ + from: localIndexes.vectors, + to: filePaths.vectors, + size, + }); + logger.log(` ${chalk.cyan('Vector store:')} ${localIndexes.vectors}`); + logger.log(` ${chalk.gray(`→ ${filePaths.vectors}`)}`); + logger.log(` ${chalk.gray(`Size: ${formatBytes(size)}`)}`); + } + + if (localIndexes.indexerState) { + const stat = await fs.stat(localIndexes.indexerState); + totalSize += stat.size; + filesToMigrate.push({ + from: localIndexes.indexerState, + to: filePaths.indexerState, + size: stat.size, + }); + logger.log(` ${chalk.cyan('Indexer state:')} ${localIndexes.indexerState}`); + logger.log(` ${chalk.gray(`→ ${filePaths.indexerState}`)}`); + logger.log(` ${chalk.gray(`Size: ${formatBytes(stat.size)}`)}`); + } + + if (localIndexes.githubState) { + const stat = await fs.stat(localIndexes.githubState); + totalSize += stat.size; + filesToMigrate.push({ + from: localIndexes.githubState, + to: filePaths.githubState, + size: stat.size, + }); + logger.log(` ${chalk.cyan('GitHub state:')} ${localIndexes.githubState}`); + logger.log(` ${chalk.gray(`→ ${filePaths.githubState}`)}`); + logger.log(` ${chalk.gray(`Size: ${formatBytes(stat.size)}`)}`); + } + + logger.log(''); + logger.log(` ${chalk.bold('Total size:')} ${formatBytes(totalSize)}`); + logger.log(` ${chalk.bold('Storage location:')} ${storagePath}`); + logger.log(''); + + if (centralizedExists) { + logger.warn('⚠️ Centralized storage already exists!'); + logger.log('Migration will merge/overwrite existing indexes.'); + logger.log(''); + } + + // Dry run mode + if (options.dryRun) { + logger.log(chalk.yellow('🔍 DRY RUN MODE - No files will be moved')); + logger.log(''); + logger.log('To actually migrate, run without --dry-run flag.'); + return; + } + + // Confirm unless --force + if (!options.force) { + logger.warn('This will move indexes to centralized storage.'); + logger.log(`Run with ${chalk.yellow('--force')} to skip this prompt.`); + logger.log(''); + process.exit(0); + } + + // Perform migration + spinner.start('Migrating indexes...'); + + for (const file of filesToMigrate) { + try { + // Ensure target directory exists + await fs.mkdir(path.dirname(file.to), { recursive: true }); + + // Move file/directory + await fs.rename(file.from, file.to); + spinner.text = `Migrated ${path.basename(file.from)}`; + } catch (error) { + spinner.fail(`Failed to migrate ${path.basename(file.from)}`); + logger.error(`Error: ${error instanceof Error ? error.message : String(error)}`); + // Continue with other files + } + } + + // Create/update metadata + try { + const existingMetadata = await loadMetadata(storagePath); + await saveMetadata(storagePath, resolvedRepoPath, { + ...existingMetadata, + migrated: { + timestamp: new Date().toISOString(), + from: resolvedRepoPath, + }, + }); + } catch (error) { + logger.debug(`Failed to update metadata: ${error}`); + } + + // Clean up empty .dev-agent directory + try { + const localDevAgentDir = path.join(resolvedRepoPath, '.dev-agent'); + const entries = await fs.readdir(localDevAgentDir); + if (entries.length === 0) { + await fs.rmdir(localDevAgentDir); + } + } catch { + // Ignore errors + } + + spinner.succeed(chalk.green('Migration completed successfully!')); + + logger.log(''); + logger.log(`✓ Indexes migrated to: ${chalk.cyan(storagePath)}`); + logger.log(`✓ ${formatBytes(totalSize)} moved to centralized storage`); + logger.log(''); + logger.log('Local indexes have been moved. Your repository is now clean!'); + logger.log(''); + } catch (error) { + spinner.fail('Migration failed'); + logger.error(`Error: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } + }); + +/** + * Info command - Show storage information + */ +storageCommand + .command('info') + .description('Show storage information and repository list') + .action(async () => { + const spinner = ora('Loading storage information...').start(); + + try { + // Load config + const config = await loadConfig(); + if (!config) { + spinner.fail('No config found'); + logger.error('Run "dev init" first to initialize dev-agent'); + process.exit(1); + return; + } + + // Resolve repository path + const repositoryPath = config.repository?.path || config.repositoryPath || process.cwd(); + const resolvedRepoPath = path.resolve(repositoryPath); + + // Get centralized storage path + const storagePath = await getStoragePath(resolvedRepoPath); + const filePaths = getStorageFilePaths(storagePath); + + spinner.stop(); + + // Check if storage exists + let storageExists = false; + let totalSize = 0; + try { + await fs.access(storagePath); + storageExists = true; + totalSize = await getDirectorySize(storagePath); + } catch { + // Storage doesn't exist yet + } + + logger.log(''); + logger.log(chalk.bold('💾 Storage Information')); + logger.log(''); + logger.log(` ${chalk.cyan('Storage Location:')} ${storagePath}`); + logger.log( + ` ${chalk.cyan('Status:')} ${storageExists ? chalk.green('Active') : chalk.gray('Not initialized')}` + ); + + if (storageExists) { + logger.log(` ${chalk.cyan('Total Size:')} ${formatBytes(totalSize)}`); + logger.log(''); + + // Show individual files + logger.log(chalk.bold('📁 Index Files:')); + logger.log(''); + + const files = [ + { name: 'Vector Store', path: filePaths.vectors }, + { name: 'Indexer State', path: filePaths.indexerState }, + { name: 'GitHub State', path: filePaths.githubState }, + { name: 'Metadata', path: filePaths.metadata }, + ]; + + for (const file of files) { + try { + const stat = await fs.stat(file.path); + const size = stat.isDirectory() ? await getDirectorySize(file.path) : stat.size; + const exists = chalk.green('✓'); + logger.log(` ${exists} ${chalk.cyan(`${file.name}:`)} ${formatBytes(size)}`); + logger.log(` ${chalk.gray(file.path)}`); + } catch { + const missing = chalk.gray('○'); + logger.log( + ` ${missing} ${chalk.gray(`${file.name}:`)} ${chalk.gray('Not found')}` + ); + } + } + + // Load and show metadata if available + try { + const metadata = await loadMetadata(storagePath); + if (metadata) { + logger.log(''); + logger.log(chalk.bold('📋 Repository Metadata:')); + logger.log(''); + if (metadata.repository?.remote) { + logger.log(` ${chalk.cyan('Remote:')} ${metadata.repository.remote}`); + } + if (metadata.repository?.branch) { + logger.log(` ${chalk.cyan('Branch:')} ${metadata.repository.branch}`); + } + if (metadata.indexed) { + logger.log( + ` ${chalk.cyan('Last Indexed:')} ${new Date(metadata.indexed.timestamp).toLocaleString()}` + ); + logger.log(` ${chalk.cyan('Files Indexed:')} ${metadata.indexed.files}`); + logger.log(` ${chalk.cyan('Components:')} ${metadata.indexed.components}`); + } + } + } catch { + // Metadata not available + } + } else { + logger.log(''); + logger.log(chalk.gray('No indexes found. Run "dev index" to create indexes.')); + } + + logger.log(''); + } catch (error) { + spinner.fail('Failed to load storage information'); + logger.error(`Error: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } + }); + +export { storageCommand }; diff --git a/packages/cli/src/utils/config.test.ts b/packages/cli/src/utils/config.test.ts index 46b4fb4..8ef7c28 100644 --- a/packages/cli/src/utils/config.test.ts +++ b/packages/cli/src/utils/config.test.ts @@ -20,8 +20,13 @@ describe('Config Utilities', () => { it('should return default configuration', () => { const config = getDefaultConfig('/test/path'); + expect(config.version).toBe('1.0'); + expect(config.repository).toBeDefined(); + expect(config.repository?.path).toBe('.'); + expect(config.repository?.excludePatterns).toContain('**/node_modules/**'); + expect(config.repository?.languages).toContain('typescript'); + // Legacy fields for backward compatibility expect(config.repositoryPath).toBe(path.resolve('/test/path')); - expect(config.vectorStorePath).toContain('.dev-agent/vectors.lance'); expect(config.embeddingModel).toBe('Xenova/all-MiniLM-L6-v2'); expect(config.dimension).toBe(384); expect(config.excludePatterns).toContain('**/node_modules/**'); @@ -39,7 +44,7 @@ describe('Config Utilities', () => { const config = getDefaultConfig(testDir); await saveConfig(config, testDir); - const configPath = path.join(testDir, '.dev-agent.json'); + const configPath = path.join(testDir, '.dev-agent', 'config.json'); const exists = await fs .access(configPath) .then(() => true) @@ -51,10 +56,13 @@ describe('Config Utilities', () => { const config = getDefaultConfig(testDir); await saveConfig(config, testDir); - const configPath = path.join(testDir, '.dev-agent.json'); + const configPath = path.join(testDir, '.dev-agent', 'config.json'); const content = await fs.readFile(configPath, 'utf-8'); const parsed = JSON.parse(content); + expect(parsed.version).toBe('1.0'); + expect(parsed.repository).toBeDefined(); + // Legacy fields for backward compatibility expect(parsed.repositoryPath).toBe(config.repositoryPath); expect(parsed.embeddingModel).toBe(config.embeddingModel); }); @@ -66,7 +74,7 @@ describe('Config Utilities', () => { await saveConfig(config, testDir); const found = await findConfigFile(testDir); - expect(found).toBe(path.join(testDir, '.dev-agent.json')); + expect(found).toBe(path.join(testDir, '.dev-agent', 'config.json')); }); it('should find config file in parent directory', async () => { @@ -77,7 +85,7 @@ describe('Config Utilities', () => { await saveConfig(config, testDir); const found = await findConfigFile(subDir); - expect(found).toBe(path.join(testDir, '.dev-agent.json')); + expect(found).toBe(path.join(testDir, '.dev-agent', 'config.json')); }); it('should return null if no config found', async () => { @@ -99,19 +107,24 @@ describe('Config Utilities', () => { const config = getDefaultConfig(testDir); await saveConfig(config, testDir); - const loaded = await loadConfig(path.join(testDir, '.dev-agent.json')); + const loaded = await loadConfig(path.join(testDir, '.dev-agent', 'config.json')); expect(loaded).toBeDefined(); + expect(loaded?.version).toBe('1.0'); + expect(loaded?.repository).toBeDefined(); + // Legacy fields for backward compatibility expect(loaded?.repositoryPath).toBe(config.repositoryPath); expect(loaded?.embeddingModel).toBe(config.embeddingModel); }); it('should return null if config not found', async () => { - const loaded = await loadConfig('/nonexistent/path/.dev-agent.json'); + const loaded = await loadConfig('/nonexistent/path/.dev-agent/config.json'); expect(loaded).toBeNull(); }); it('should handle invalid JSON gracefully', async () => { - const invalidPath = path.join(testDir, '.dev-agent-invalid.json'); + const invalidDir = path.join(testDir, '.dev-agent-invalid'); + await fs.mkdir(invalidDir, { recursive: true }); + const invalidPath = path.join(invalidDir, 'config.json'); await fs.writeFile(invalidPath, 'invalid json{{{', 'utf-8'); const loaded = await loadConfig(invalidPath); diff --git a/packages/mcp-server/bin/dev-agent-mcp.ts b/packages/mcp-server/bin/dev-agent-mcp.ts index f512d80..dd76914 100644 --- a/packages/mcp-server/bin/dev-agent-mcp.ts +++ b/packages/mcp-server/bin/dev-agent-mcp.ts @@ -4,7 +4,13 @@ * Starts the MCP server with stdio transport for AI tools (Claude, Cursor, etc.) */ -import { RepositoryIndexer } from '@lytics/dev-agent-core'; +import { + ensureStorageDirectory, + getStorageFilePaths, + getStoragePath, + RepositoryIndexer, + saveMetadata, +} from '@lytics/dev-agent-core'; import { ExplorerAgent, PlannerAgent, @@ -22,20 +28,87 @@ import { MCPServer } from '../src/server/mcp-server'; // Get config from environment const repositoryPath = process.env.REPOSITORY_PATH || process.cwd(); -const vectorStorePath = - process.env.VECTOR_STORE_PATH || `${repositoryPath}/.dev-agent/vectors.lance`; const logLevel = (process.env.LOG_LEVEL as 'debug' | 'info' | 'warn' | 'error') || 'info'; +// Lazy-loaded indexer +let indexer: RepositoryIndexer | undefined; +let lastAccessed = Date.now(); +const IDLE_TIMEOUT = 5 * 60 * 1000; // 5 minutes + +/** + * Ensure indexer is loaded (lazy loading) + * This will be called on first tool use + */ +async function _ensureIndexer(): Promise { + if (!indexer) { + // Get centralized storage path + const storagePath = await getStoragePath(repositoryPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); + + // Initialize repository indexer with centralized storage + indexer = new RepositoryIndexer({ + repositoryPath, + vectorStorePath: filePaths.vectors, + statePath: filePaths.indexerState, + }); + + await indexer.initialize(); + + // Update metadata + await saveMetadata(storagePath, repositoryPath); + + console.error(`[MCP] Loaded indexes from ${storagePath}`); + } + + lastAccessed = Date.now(); + return indexer; +} + +/** + * Auto-unload indexer after idle period + * TODO: Enable idle monitoring in future iteration + */ +function _startIdleMonitor(): void { + setInterval(() => { + const idleTime = Date.now() - lastAccessed; + + if (idleTime > IDLE_TIMEOUT && indexer) { + indexer + .close() + .then(() => { + indexer = undefined; + const idleMinutes = Math.floor(idleTime / 60000); + console.error(`[MCP] Unloaded indexes (idle timeout: ${idleMinutes} minutes)`); + }) + .catch((error) => { + console.error('[MCP] Error unloading indexes:', error); + }); + } + }, 60000); // Check every minute +} + async function main() { try { - // Initialize repository indexer + // Get centralized storage paths + const storagePath = await getStoragePath(repositoryPath); + await ensureStorageDirectory(storagePath); + const filePaths = getStorageFilePaths(storagePath); + + // Initialize repository indexer with centralized storage + // TODO: Make this truly lazy (only initialize on first tool call) + // For now, initialize eagerly but use centralized storage const indexer = new RepositoryIndexer({ repositoryPath, - vectorStorePath, + vectorStorePath: filePaths.vectors, + statePath: filePaths.indexerState, }); await indexer.initialize(); + // Update metadata + await saveMetadata(storagePath, repositoryPath); + // Create and configure the subagent coordinator const coordinator = new SubagentCoordinator({ maxConcurrentTasks: 5, @@ -61,7 +134,7 @@ async function main() { const statusAdapter = new StatusAdapter({ repositoryIndexer: indexer, repositoryPath, - vectorStorePath, + vectorStorePath: filePaths.vectors, defaultSection: 'summary', }); @@ -83,8 +156,8 @@ async function main() { const githubAdapter = new GitHubAdapter({ repositoryPath, // GitHubIndexer will be lazily initialized on first use - vectorStorePath: `${vectorStorePath}-github`, - statePath: `${repositoryPath}/.dev-agent/github-state.json`, + vectorStorePath: `${filePaths.vectors}-github`, + statePath: filePaths.githubState, defaultLimit: 10, defaultFormat: 'compact', }); diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..6a2a3a9 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,57 @@ +# Scripts + +Utility scripts for managing the dev-agent repository and GitHub issues. + +## gh-issue-add-subs.sh + +Add multiple sub-issues to a parent issue using GitHub's GraphQL API. + +### Usage + +```bash +./scripts/gh-issue-add-subs.sh ... +``` + +### Examples + +```bash +# Add issues #52, #53, #54 as sub-issues of #31 +./scripts/gh-issue-add-subs.sh 31 52 53 54 + +# Add a single sub-issue +./scripts/gh-issue-add-subs.sh 31 52 +``` + +### Requirements + +- GitHub CLI (`gh`) installed and authenticated +- Issues must exist in the current repository +- `jq` (optional, for better error formatting) + +### How It Works + +1. Fetches the parent issue ID using `gh issue view` +2. Fetches each child issue ID +3. Builds a batched GraphQL mutation with aliases (`add1`, `add2`, etc.) +4. Executes all mutations in a single GraphQL request +5. Displays results for each relationship created + +### Exit Codes + +- `0` - Success +- `1` - Missing required arguments +- `2` - Failed to get parent issue ID +- `3` - Failed to get child issue ID +- `4` - GraphQL mutation failed + +### Notes + +- The script uses GraphQL aliases to batch multiple `addSubIssue` mutations in a single request +- If a child issue already has a parent, GitHub will return an error (handled gracefully) +- All operations are performed atomically - if one fails, the entire batch fails + +### Related GitHub Features + +- [GitHub Issue Relationships](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/adding-sub-issues) +- [GitHub GraphQL API - addSubIssue](https://docs.github.com/en/graphql/reference/mutations#addsubissue) + diff --git a/scripts/gh-issue-add-subs.sh b/scripts/gh-issue-add-subs.sh new file mode 100755 index 0000000..fd9e4a5 --- /dev/null +++ b/scripts/gh-issue-add-subs.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# +# gh-issue-add-subs - Add multiple sub-issues to a parent issue +# +# Usage: +# gh-issue-add-subs ... +# +# Description: +# This script adds multiple existing issues as sub-issues to a parent issue +# using GitHub's GraphQL API. All relationships are created in a single +# batched GraphQL request for efficiency. +# +# Examples: +# # Add issues #52, #53, #54 as sub-issues of #31 +# gh-issue-add-subs 31 52 53 54 +# +# # Add a single sub-issue +# gh-issue-add-subs 31 52 +# +# Requirements: +# - GitHub CLI (gh) installed and authenticated +# - Issues must exist in the current repository +# +# Exit codes: +# 0 - Success +# 1 - Missing required arguments +# 2 - Failed to get parent issue ID +# 3 - Failed to get child issue ID +# 4 - GraphQL mutation failed + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Print usage +usage() { + cat << EOF +Usage: $0 ... + +Add multiple sub-issues to a parent issue using GitHub's GraphQL API. + +Arguments: + parent-issue-number The issue number of the parent issue + child-issue-number... One or more issue numbers to add as sub-issues + +Examples: + $0 31 52 53 54 + $0 31 52 + +EOF +} + +# Check if GitHub CLI is available +if ! command -v gh &> /dev/null; then + echo -e "${RED}Error: GitHub CLI (gh) is not installed${NC}" >&2 + echo "Install it from https://cli.github.com/" >&2 + exit 1 +fi + +# Check arguments +if [ $# -lt 2 ]; then + echo -e "${RED}Error: Missing required arguments${NC}" >&2 + usage + exit 1 +fi + +parent=$1 +shift +children=("$@") + +echo -e "${YELLOW}Adding ${#children[@]} sub-issue(s) to issue #${parent}...${NC}" + +# Get parent issue ID +echo "Fetching parent issue #${parent}..." +parent_id=$(gh issue view "$parent" --json id -q .id 2>/dev/null || true) + +if [ -z "$parent_id" ]; then + echo -e "${RED}Error: Failed to get parent issue #${parent}${NC}" >&2 + echo "Make sure the issue exists and you have access to it." >&2 + exit 2 +fi + +# Build GraphQL mutation with aliases for each child +mutation="mutation {" +i=1 +child_ids=() + +for child in "${children[@]}"; do + echo "Fetching child issue #${child}..." + child_id=$(gh issue view "$child" --json id -q .id 2>/dev/null || true) + + if [ -z "$child_id" ]; then + echo -e "${RED}Error: Failed to get child issue #${child}${NC}" >&2 + echo "Make sure the issue exists and you have access to it." >&2 + exit 3 + fi + + child_ids+=("$child_id") + + # Add mutation with unique alias + mutation+=" + add$i: addSubIssue(input: { + issueId: \"$parent_id\" + subIssueId: \"$child_id\" + }) { + parentIssue { + number + title + } + subIssue { + number + title + } + }" + ((i++)) +done + +mutation+=" +}" + +# Execute GraphQL mutation +echo -e "${YELLOW}Creating relationships...${NC}" +response=$(gh api graphql -f query="$mutation" 2>&1) + +# Check for errors in response +if echo "$response" | grep -q '"errors"'; then + echo -e "${RED}Error: GraphQL mutation failed${NC}" >&2 + echo "$response" | jq '.' >&2 || echo "$response" >&2 + exit 4 +fi + +# Parse and display results +echo -e "${GREEN}✓ Successfully added sub-issues:${NC}" +i=1 +for child in "${children[@]}"; do + result=$(echo "$response" | jq -r ".data.add$i.subIssue.number // empty" 2>/dev/null || echo "") + if [ -n "$result" ]; then + title=$(echo "$response" | jq -r ".data.add$i.subIssue.title // \"\"" 2>/dev/null || echo "") + echo -e " ${GREEN}✓${NC} Issue #${child} → Sub-issue of #${parent}" + if [ -n "$title" ]; then + echo " \"$title\"" + fi + else + echo -e " ${YELLOW}⚠${NC} Issue #${child} (may already be a sub-issue)" + fi + ((i++)) +done + +echo -e "\n${GREEN}Done!${NC} View issue #${parent} on GitHub to see the relationships." +