diff --git a/bin/sidecar.js b/bin/sidecar.js index ee95787..bad12fb 100755 --- a/bin/sidecar.js +++ b/bin/sidecar.js @@ -121,7 +121,8 @@ async function main() { */ async function handleStart(args) { // Resolve model alias or use config default before validation - const { resolveModel } = require('../src/utils/config'); + const { resolveModel, detectFallback, loadConfig } = require('../src/utils/config'); + const rawAlias = args.model; try { args.model = resolveModel(args.model); } catch (err) { @@ -129,6 +130,28 @@ async function handleStart(args) { process.exit(1); } + // Determine the alias used (explicit or config default) + let alias = rawAlias; + if (alias === undefined) { + const cfg = loadConfig(); + if (cfg && cfg.default && !cfg.default.includes('/')) { + alias = cfg.default; + } + } + + // Validate direct-API fallback models exist on the provider (opt-in via --validate-model) + if (args['validate-model'] && alias && detectFallback(alias, args.model)) { + const { validateDirectModel } = require('../src/utils/model-validator'); + try { + args.model = await validateDirectModel(args.model, alias, { + headless: args['no-ui'] || !process.stdin.isTTY + }); + } catch (err) { + console.error(err.message); + process.exit(1); + } + } + // Normalize agent: --agent takes precedence, otherwise use --mode // (Must happen before validation so --mode alias is also validated) args.agent = args.agent || args.mode; diff --git a/src/cli.js b/src/cli.js index ad55910..f928b68 100644 --- a/src/cli.js +++ b/src/cli.js @@ -94,7 +94,8 @@ function isBooleanFlag(key) { 'json', 'version', 'help', - 'api-keys' + 'api-keys', + 'validate-model' ]; return booleanFlags.includes(key); } @@ -313,6 +314,7 @@ Options for 'start': --mcp-config Path to opencode.json with MCP config --no-mcp Don't inherit MCP servers from parent LLM --exclude-mcp Exclude specific MCP server (repeatable) + --validate-model Verify model exists on provider API (opt-in) --position Window position: right (default), left, center Options for 'list': diff --git a/src/utils/api-key-store.js b/src/utils/api-key-store.js index ed21f9f..cffaa5d 100644 --- a/src/utils/api-key-store.js +++ b/src/utils/api-key-store.js @@ -11,7 +11,8 @@ const PROVIDER_ENV_MAP = { openrouter: 'OPENROUTER_API_KEY', google: 'GEMINI_API_KEY', openai: 'OPENAI_API_KEY', - anthropic: 'ANTHROPIC_API_KEY' + anthropic: 'ANTHROPIC_API_KEY', + deepseek: 'DEEPSEEK_API_KEY' }; /** Validation endpoints per provider */ @@ -34,6 +35,10 @@ const VALIDATION_ENDPOINTS = { google: { url: 'https://generativelanguage.googleapis.com/v1beta/models', authHeader: () => ({}) + }, + deepseek: { + url: 'https://api.deepseek.com/models', + authHeader: (key) => ({ 'Authorization': `Bearer ${key}` }) } }; diff --git a/src/utils/config.js b/src/utils/config.js index e2003f3..185aa91 100644 --- a/src/utils/config.js +++ b/src/utils/config.js @@ -9,11 +9,10 @@ const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); +const { PROVIDER_ENV_MAP } = require('./api-key-store'); +const { logger } = require('./logger'); -/** - * Default model alias map - * Maps short alias names to full OpenRouter model identifiers. - */ +/** Default model alias map — short names to full OpenRouter model identifiers */ const DEFAULT_ALIASES = { 'gemini': 'openrouter/google/gemini-3.1-flash-lite-preview', 'gemini-pro': 'openrouter/google/gemini-3.1-pro-preview', @@ -37,10 +36,7 @@ const DEFAULT_ALIASES = { 'seed': 'openrouter/bytedance-seed/seed-2.0-mini', }; -/** - * Get the sidecar configuration directory path - * @returns {string} Config directory path - */ +/** @returns {string} Config directory path */ function getConfigDir() { if (process.env.SIDECAR_CONFIG_DIR) { const resolved = path.resolve(process.env.SIDECAR_CONFIG_DIR); @@ -53,18 +49,12 @@ function getConfigDir() { return path.join(homeDir, '.config', 'sidecar'); } -/** - * Get the path to the config.json file - * @returns {string} Full path to config.json - */ +/** @returns {string} Full path to config.json */ function getConfigPath() { return path.join(getConfigDir(), 'config.json'); } -/** - * Load and parse the config file - * @returns {object|null} Parsed config data, or null if missing/invalid - */ +/** @returns {object|null} Parsed config data, or null if missing/invalid */ function loadConfig() { const configPath = getConfigPath(); try { @@ -81,10 +71,7 @@ function loadConfig() { } } -/** - * Save config data to disk, creating the directory if needed - * @param {object} configData - Configuration object to persist - */ +/** Save config data to disk, creating the directory if needed */ function saveConfig(configData) { const configDir = getConfigDir(); fs.mkdirSync(configDir, { recursive: true, mode: 0o700 }); @@ -92,14 +79,29 @@ function saveConfig(configData) { fs.writeFileSync(configPath, JSON.stringify(configData, null, 2), { mode: 0o600 }); } -/** - * Get the default alias map - * @returns {object} Map of alias name to full model identifier - */ +/** @returns {object} Copy of the default alias map */ function getDefaultAliases() { return { ...DEFAULT_ALIASES }; } +/** Strip openrouter/ prefix when direct provider API key is available but OPENROUTER_API_KEY is not */ +function applyDirectApiFallback(model) { + if (!model.startsWith('openrouter/') || process.env.OPENROUTER_API_KEY) { + return model; + } + const direct = model.slice('openrouter/'.length); + const envVar = PROVIDER_ENV_MAP[direct.split('/')[0]]; + if (envVar && process.env[envVar]) { + logger.warn({ msg: 'Using direct provider API (OPENROUTER_API_KEY not set)', original: model, resolved: direct }); + process.stderr.write( + `Notice: Using direct ${direct.split('/')[0]} API (OPENROUTER_API_KEY not set). ` + + 'Use --validate-model to verify model availability.\n' + ); + return direct; + } + return model; +} + /** * Resolve a model argument to a full model identifier * @@ -128,7 +130,7 @@ function resolveModel(modelArg) { // Try to resolve as alias (user config + defaults) if (effectiveAliases[modelArg] !== undefined) { - return effectiveAliases[modelArg]; + return applyDirectApiFallback(effectiveAliases[modelArg]); } // Unknown alias @@ -153,7 +155,7 @@ function resolveModel(modelArg) { // Default is an alias - resolve via user config + defaults if (effectiveAliases[defaultValue] !== undefined) { - return effectiveAliases[defaultValue]; + return applyDirectApiFallback(effectiveAliases[defaultValue]); } // Default alias not found anywhere @@ -162,10 +164,7 @@ function resolveModel(modelArg) { ); } -/** - * Compute SHA-256 hash of the config file content (first 8 hex chars) - * @returns {string|null} 8-char hex hash, or null if no config file - */ +/** @returns {string|null} 8-char hex hash of config file, or null if missing */ function computeConfigHash() { const configPath = getConfigPath(); try { @@ -179,10 +178,7 @@ function computeConfigHash() { } } -/** - * Format aliases as a markdown table with (default) marker - * @returns {string} Markdown-formatted alias table, or empty string if no aliases - */ +/** @returns {string} Markdown alias table with (default) marker, or empty string */ function buildAliasTable() { const config = loadConfig(); if (!config || !config.aliases || Object.keys(config.aliases).length === 0) { @@ -203,11 +199,7 @@ function buildAliasTable() { return lines.join('\n'); } -/** - * Check whether the config file has changed compared to a known hash - * @param {string|null} currentHash - Previously known hash to compare - * @returns {{changed: boolean, newHash: string|null, updateData?: string}} - */ +/** Check whether the config file has changed compared to a known hash */ function checkConfigChanged(currentHash) { const newHash = computeConfigHash(); @@ -281,6 +273,13 @@ function buildProviderModels() { return providers; } +/** Detect if direct API fallback was applied during alias resolution */ +function detectFallback(alias, resolvedModel) { + if (!alias || alias.includes('/')) { return false; } + const val = getEffectiveAliases()[alias]; + return !!(val && val.startsWith('openrouter/') && !resolvedModel.startsWith('openrouter/')); +} + module.exports = { getConfigDir, getConfigPath, @@ -288,6 +287,7 @@ module.exports = { saveConfig, getDefaultAliases, resolveModel, + detectFallback, computeConfigHash, buildAliasTable, checkConfigChanged, diff --git a/src/utils/model-validator.js b/src/utils/model-validator.js new file mode 100644 index 0000000..564c838 --- /dev/null +++ b/src/utils/model-validator.js @@ -0,0 +1,141 @@ +/** + * Model Validator + * + * Validates that a direct-API fallback model exists on the provider. + * When a model is not found, prompts the user to pick an alternative + * (interactive) or fails with available models (headless). + */ + +const readline = require('readline'); +const { fetchModelsFromProvider } = require('./model-fetcher'); +const { readApiKeyValues } = require('./api-key-store'); +const { loadConfig, saveConfig, getConfigPath } = require('./config'); +const { logger } = require('./logger'); + +/** Alias-to-search-term mapping for filtering provider model lists */ +const ALIAS_SEARCH_TERMS = { + 'gemini': 'gemini', 'gemini-pro': 'gemini', + 'gpt': 'gpt', 'gpt-pro': 'gpt', 'codex': 'gpt', + 'claude': 'claude', 'sonnet': 'claude', 'opus': 'claude', 'haiku': 'claude', + 'deepseek': 'deepseek', +}; + +/** + * Validate a direct-API fallback model exists on the provider. + * Returns silently if valid. On failure: prompts (interactive) or throws (headless). + * + * @param {string} resolvedModel - e.g. 'google/gemini-3.1-flash-lite-preview' + * @param {string} alias - Original alias name (e.g. 'gemini') + * @param {object} [options] + * @param {boolean} [options.headless] - If true, throw instead of prompting + * @returns {Promise} Confirmed model string + */ +async function validateDirectModel(resolvedModel, alias, options = {}) { + const parts = resolvedModel.split('/'); + if (parts.length < 2) { return resolvedModel; } + + const provider = parts[0]; + const modelId = parts.slice(1).join('/'); + + const keys = readApiKeyValues(); + const providerKey = keys[provider]; + if (!providerKey) { return resolvedModel; } + + let models; + try { + models = await fetchModelsFromProvider(provider, providerKey); + } catch (err) { + logger.debug({ msg: 'Model fetch failed, skipping validation', error: err.message }); + return resolvedModel; + } + + if (!models || models.length === 0) { return resolvedModel; } + + const found = models.some(m => m.id === resolvedModel || m.id === modelId); + if (found) { return resolvedModel; } + + const relevant = filterRelevantModels(models, alias); + + if (options.headless || !process.stdin.isTTY) { + const list = relevant.slice(0, 10).map(m => ` ${m.id}`).join('\n'); + throw new Error( + `Model '${modelId}' not found on ${provider} API.\n` + + `Available models:\n${list}\n` + + `Fix with: sidecar setup --add-alias ${alias}=${relevant[0]?.id || 'provider/model'}` + ); + } + + return promptModelSelection(relevant, alias, provider, modelId); +} + +/** + * Filter models to those relevant to the alias + * @param {Array<{id: string, name: string}>} models + * @param {string} alias - e.g. 'gemini', 'gpt', 'opus' + * @returns {Array<{id: string, name: string}>} Filtered, sorted, max 15 + */ +function filterRelevantModels(models, alias) { + const term = (ALIAS_SEARCH_TERMS[alias] || alias).toLowerCase(); + + let filtered = models.filter(m => + m.id.toLowerCase().includes(term) || + m.name.toLowerCase().includes(term) + ); + + if (filtered.length === 0) { filtered = models; } + + filtered.sort((a, b) => a.name.localeCompare(b.name)); + return filtered.slice(0, 15); +} + +/** Interactive prompt — ask user to pick from available models */ +async function promptModelSelection(models, alias, provider, failedModelId) { + process.stderr.write(`\n Model '${failedModelId}' not found on ${provider} API.\n`); + process.stderr.write(' Available models:\n'); + models.forEach((m, i) => { + const label = (m.name && m.name !== m.id) ? `${m.name} (${m.id})` : m.id; + process.stderr.write(` ${i + 1}. ${label}\n`); + }); + process.stderr.write('\n'); + + const rl = readline.createInterface({ input: process.stdin, output: process.stderr }); + + const answer = await new Promise(resolve => { + rl.question(` Select a model (1-${models.length}) or press Enter to cancel: `, resolve); + }); + rl.close(); + + const idx = parseInt(answer, 10) - 1; + if (isNaN(idx) || idx < 0 || idx >= models.length) { + throw new Error('Model selection cancelled.'); + } + + const selected = models[idx]; + const newModel = selected.id; + + let config = loadConfig(); + if (!config) { + const fs = require('fs'); + const configPath = getConfigPath(); + if (fs.existsSync(configPath)) { + throw new Error( + `Cannot save model selection: config file at ${configPath} is malformed. ` + + 'Fix it manually or run \'sidecar setup\'.' + ); + } + config = {}; + } + if (!config.aliases) { config.aliases = {}; } + config.aliases[alias] = newModel; + try { + saveConfig(config); + process.stderr.write(` Saved: ${alias} → ${newModel}\n`); + } catch (err) { + process.stderr.write(` Warning: Could not save selection (${err.message}). Using for this session only.\n`); + } + process.stderr.write(` (To change later: sidecar setup --add-alias ${alias}=...)\n\n`); + + return newModel; +} + +module.exports = { validateDirectModel, filterRelevantModels }; diff --git a/tests/api-key-store.test.js b/tests/api-key-store.test.js index 9e1b6a7..1e58dca 100644 --- a/tests/api-key-store.test.js +++ b/tests/api-key-store.test.js @@ -54,8 +54,8 @@ describe('api-key-store', () => { expect(PROVIDER_ENV_MAP.anthropic).toBe('ANTHROPIC_API_KEY'); }); - it('should have exactly 4 providers', () => { - expect(Object.keys(PROVIDER_ENV_MAP)).toHaveLength(4); + it('should have exactly 5 providers', () => { + expect(Object.keys(PROVIDER_ENV_MAP)).toHaveLength(5); }); }); diff --git a/tests/cli.test.js b/tests/cli.test.js index 864ceae..b39629e 100644 --- a/tests/cli.test.js +++ b/tests/cli.test.js @@ -301,6 +301,20 @@ describe('CLI Argument Parser', () => { }); }); + describe('--validate-model flag', () => { + it('should parse --validate-model as boolean flag', () => { + const result = parseArgs(['start', '--validate-model', '--model', 'gemini', '--prompt', 'test']); + expect(result['validate-model']).toBe(true); + expect(result.model).toBe('gemini'); + }); + + it('should not consume the next argument as a value', () => { + const result = parseArgs(['start', '--validate-model', '--prompt', 'test']); + expect(result['validate-model']).toBe(true); + expect(result.prompt).toBe('test'); + }); + }); + describe('--thinking option', () => { it('should parse --thinking option with valid effort level', () => { const result = parseArgs(['start', '--thinking', 'low']); diff --git a/tests/config.test.js b/tests/config.test.js index e6c623d..294d898 100644 --- a/tests/config.test.js +++ b/tests/config.test.js @@ -18,6 +18,12 @@ describe('Sidecar Config Module', () => { tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sidecar-config-test-')); originalEnv = { ...process.env }; process.env.SIDECAR_CONFIG_DIR = tempDir; + // Clear API keys to ensure deterministic fallback behavior + delete process.env.OPENROUTER_API_KEY; + delete process.env.GEMINI_API_KEY; + delete process.env.OPENAI_API_KEY; + delete process.env.ANTHROPIC_API_KEY; + delete process.env.DEEPSEEK_API_KEY; jest.resetModules(); }); @@ -324,6 +330,128 @@ describe('Sidecar Config Module', () => { ); }); + describe('resolveModel - direct API fallback', () => { + let stderrSpy; + beforeEach(() => { + stderrSpy = jest.spyOn(process.stderr, 'write').mockImplementation(() => true); + }); + afterEach(() => { + stderrSpy.mockRestore(); + }); + + it('should fall back to google/ when GEMINI_API_KEY is set but OPENROUTER_API_KEY is not', () => { + process.env.GEMINI_API_KEY = 'test-gemini-key'; + jest.resetModules(); + const config = loadModule(); + const result = config.resolveModel('gemini'); + expect(result).toBe('google/gemini-3.1-flash-lite-preview'); + }); + + it('should fall back to openai/ when OPENAI_API_KEY is set but OPENROUTER_API_KEY is not', () => { + process.env.OPENAI_API_KEY = 'test-openai-key'; + jest.resetModules(); + const config = loadModule(); + const result = config.resolveModel('gpt'); + expect(result).toBe('openai/gpt-5.4'); + }); + + it('should fall back to anthropic/ when ANTHROPIC_API_KEY is set but OPENROUTER_API_KEY is not', () => { + process.env.ANTHROPIC_API_KEY = 'test-anthropic-key'; + jest.resetModules(); + const config = loadModule(); + const result = config.resolveModel('opus'); + expect(result).toBe('anthropic/claude-opus-4.6'); + }); + + it('should fall back to deepseek/ when DEEPSEEK_API_KEY is set but OPENROUTER_API_KEY is not', () => { + process.env.DEEPSEEK_API_KEY = 'test-deepseek-key'; + jest.resetModules(); + const config = loadModule(); + const result = config.resolveModel('deepseek'); + expect(result).toBe('deepseek/deepseek-v3.2'); + }); + + it('should prefer OpenRouter when both OPENROUTER_API_KEY and direct key are set', () => { + process.env.OPENROUTER_API_KEY = 'test-openrouter-key'; + process.env.GEMINI_API_KEY = 'test-gemini-key'; + jest.resetModules(); + const config = loadModule(); + const result = config.resolveModel('gemini'); + expect(result).toBe('openrouter/google/gemini-3.1-flash-lite-preview'); + }); + + it('should return openrouter path when neither key is set', () => { + const config = loadModule(); + const result = config.resolveModel('gemini'); + expect(result).toBe('openrouter/google/gemini-3.1-flash-lite-preview'); + }); + + it('should not apply fallback to explicit model strings with slash', () => { + process.env.GEMINI_API_KEY = 'test-gemini-key'; + jest.resetModules(); + const config = loadModule(); + const result = config.resolveModel('openrouter/google/gemini-3.1-flash-lite-preview'); + expect(result).toBe('openrouter/google/gemini-3.1-flash-lite-preview'); + }); + + it('should not apply fallback for providers without direct key mapping', () => { + const config = loadModule(); + const result = config.resolveModel('qwen'); + expect(result).toBe('openrouter/qwen/qwen3.5-397b-a17b'); + }); + + it('should apply fallback to default alias resolution', () => { + process.env.GEMINI_API_KEY = 'test-gemini-key'; + const data = { default: 'gemini', aliases: {} }; + fs.writeFileSync(path.join(tempDir, 'config.json'), JSON.stringify(data)); + jest.resetModules(); + const config = loadModule(); + const result = config.resolveModel(undefined); + expect(result).toBe('google/gemini-3.1-flash-lite-preview'); + }); + + it('should not apply fallback to explicit default model strings', () => { + process.env.GEMINI_API_KEY = 'test-gemini-key'; + const data = { default: 'openrouter/google/gemini-3.1-flash-lite-preview', aliases: {} }; + fs.writeFileSync(path.join(tempDir, 'config.json'), JSON.stringify(data)); + jest.resetModules(); + const config = loadModule(); + const result = config.resolveModel(undefined); + expect(result).toBe('openrouter/google/gemini-3.1-flash-lite-preview'); + }); + }); + + describe('detectFallback', () => { + it('should return true when alias resolved via fallback', () => { + process.env.GEMINI_API_KEY = 'test-key'; + jest.resetModules(); + const config = loadModule(); + expect(config.detectFallback('gemini', 'google/gemini-3.1-flash-lite-preview')).toBe(true); + }); + + it('should return false when alias resolved via OpenRouter', () => { + process.env.OPENROUTER_API_KEY = 'test-key'; + jest.resetModules(); + const config = loadModule(); + expect(config.detectFallback('gemini', 'openrouter/google/gemini-3.1-flash-lite-preview')).toBe(false); + }); + + it('should return false for explicit model strings with slash', () => { + const config = loadModule(); + expect(config.detectFallback('openrouter/google/gemini', 'openrouter/google/gemini')).toBe(false); + }); + + it('should return false for unknown aliases', () => { + const config = loadModule(); + expect(config.detectFallback('nonexistent', 'some/model')).toBe(false); + }); + + it('should return false when alias is undefined', () => { + const config = loadModule(); + expect(config.detectFallback(undefined, 'google/gemini')).toBe(false); + }); + }); + describe('computeConfigHash', () => { it('should return null when no config file exists', () => { const config = loadModule(); diff --git a/tests/mcp-server.test.js b/tests/mcp-server.test.js index 768a16c..005cd30 100644 --- a/tests/mcp-server.test.js +++ b/tests/mcp-server.test.js @@ -26,7 +26,7 @@ describe('MCP spawn arg building', () => { }), })); const { handlers: h } = require('../src/mcp-server'); - const result = await h.sidecar_start({ prompt: 'test task', noUi: true }, '/tmp'); + const result = await h.sidecar_start({ prompt: 'test task', noUi: true, model: 'google/gemini-test' }, '/tmp'); const { taskId } = JSON.parse(result.content[0].text); const idx = capturedArgs.indexOf('--task-id'); expect(idx).toBeGreaterThan(-1); @@ -44,7 +44,7 @@ describe('MCP spawn arg building', () => { }), })); const { handlers: h } = require('../src/mcp-server'); - await h.sidecar_start({ prompt: 'test task', noUi: true }, '/tmp'); + await h.sidecar_start({ prompt: 'test task', noUi: true, model: 'google/gemini-test' }, '/tmp'); const idx = capturedArgs.indexOf('--client'); expect(idx).toBeGreaterThan(-1); expect(capturedArgs[idx + 1]).toBe('cowork'); @@ -61,7 +61,7 @@ describe('MCP spawn arg building', () => { }), })); const { handlers: h } = require('../src/mcp-server'); - await h.sidecar_start({ prompt: 'test task', noUi: true, parentSession: 'f58f2782-fc8c-41bc-afbc-e0c130b91aaf' }, '/tmp'); + await h.sidecar_start({ prompt: 'test task', noUi: true, model: 'google/gemini-test', parentSession: 'f58f2782-fc8c-41bc-afbc-e0c130b91aaf' }, '/tmp'); const idx = capturedArgs.indexOf('--session-id'); expect(idx).toBeGreaterThan(-1); expect(capturedArgs[idx + 1]).toBe('f58f2782-fc8c-41bc-afbc-e0c130b91aaf'); @@ -78,7 +78,7 @@ describe('MCP spawn arg building', () => { }), })); const { handlers: h } = require('../src/mcp-server'); - await h.sidecar_start({ prompt: 'test task', noUi: true, timeout: 30 }, '/tmp'); + await h.sidecar_start({ prompt: 'test task', noUi: true, model: 'google/gemini-test', timeout: 30 }, '/tmp'); const idx = capturedArgs.indexOf('--timeout'); expect(idx).toBeGreaterThan(-1); expect(capturedArgs[idx + 1]).toBe('30'); @@ -780,7 +780,7 @@ describe('MCP Server Handlers', () => { }), })); const { handlers: h } = require('../src/mcp-server'); - const result = await h.sidecar_start({ prompt: 'analyze auth', noUi: false }, '/tmp'); + const result = await h.sidecar_start({ prompt: 'analyze auth', noUi: false, model: 'google/gemini-test' }, '/tmp'); const parsed = JSON.parse(result.content[0].text); expect(parsed.mode).toBe('interactive'); expect(parsed.message).toContain('Do NOT poll'); @@ -798,7 +798,7 @@ describe('MCP Server Handlers', () => { }), })); const { handlers: h } = require('../src/mcp-server'); - const result = await h.sidecar_start({ prompt: 'implement feature', noUi: true }, '/tmp'); + const result = await h.sidecar_start({ prompt: 'implement feature', noUi: true, model: 'google/gemini-test' }, '/tmp'); const parsed = JSON.parse(result.content[0].text); expect(parsed.mode).toBe('headless'); expect(parsed.message).toContain('at least 30s'); @@ -811,7 +811,7 @@ describe('MCP Server Handlers', () => { spawn: jest.fn(() => ({ pid: 12345, unref: jest.fn() })), })); const { handlers: h } = require('../src/mcp-server'); - const result = await h.sidecar_start({ prompt: 'test task', noUi: true }, '/tmp'); + const result = await h.sidecar_start({ prompt: 'test task', noUi: true, model: 'google/gemini-test' }, '/tmp'); expect(result.content).toHaveLength(2); expect(result.content[1].text).toContain(''); expect(result.content[1].text).toContain('at least 30s'); @@ -824,7 +824,7 @@ describe('MCP Server Handlers', () => { spawn: jest.fn(() => ({ pid: 12345, unref: jest.fn() })), })); const { handlers: h } = require('../src/mcp-server'); - const result = await h.sidecar_start({ prompt: 'analyze auth', noUi: false }, '/tmp'); + const result = await h.sidecar_start({ prompt: 'analyze auth', noUi: false, model: 'google/gemini-test' }, '/tmp'); expect(result.content).toHaveLength(1); expect(result.content[0].text).not.toContain(''); }); @@ -1176,7 +1176,7 @@ describe('sidecar_start stderr capture', () => { // Ensure real fs is used (clear any leaked mock from prior tests) jest.doMock('fs', () => jest.requireActual('fs')); const { handlers: h } = require('../src/mcp-server'); - await h.sidecar_start({ prompt: 'test task', noUi: true }, tmpDir); + await h.sidecar_start({ prompt: 'test task', noUi: true, model: 'google/gemini-test' }, tmpDir); }); // stdio[2] should be a number (file descriptor), not 'ignore' expect(typeof capturedOpts.stdio[2]).toBe('number'); diff --git a/tests/model-validator.test.js b/tests/model-validator.test.js new file mode 100644 index 0000000..81ab1a7 --- /dev/null +++ b/tests/model-validator.test.js @@ -0,0 +1,286 @@ +/** + * Model Validator Tests + * + * Tests for direct-API model validation, filtering, and user prompting. + */ + +jest.mock('../src/utils/model-fetcher'); +jest.mock('../src/utils/api-key-store'); +jest.mock('../src/utils/config'); +jest.mock('../src/utils/logger', () => ({ + logger: { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn() } +})); + +const { fetchModelsFromProvider } = require('../src/utils/model-fetcher'); +const { readApiKeyValues } = require('../src/utils/api-key-store'); +const { loadConfig, saveConfig, getConfigPath } = require('../src/utils/config'); + +const MOCK_GOOGLE_MODELS = [ + { id: 'google/gemini-2.5-flash', name: 'Gemini 2.5 Flash' }, + { id: 'google/gemini-2.5-pro', name: 'Gemini 2.5 Pro' }, + { id: 'google/gemini-2.0-flash', name: 'Gemini 2.0 Flash' }, + { id: 'google/gemini-1.5-pro', name: 'Gemini 1.5 Pro' }, + { id: 'google/text-embedding-004', name: 'Text Embedding 004' }, +]; + +const MOCK_OPENAI_MODELS = [ + { id: 'openai/gpt-4o', name: 'gpt-4o' }, + { id: 'openai/gpt-4-turbo', name: 'gpt-4-turbo' }, + { id: 'openai/o3-mini', name: 'o3-mini' }, +]; + +describe('Model Validator', () => { + let validateDirectModel; + let filterRelevantModels; + const origIsTTY = process.stdin.isTTY; + + afterAll(() => { + process.stdin.isTTY = origIsTTY; + }); + + beforeEach(() => { + jest.resetModules(); + jest.clearAllMocks(); + + // Re-require after reset to get fresh module with mocks + jest.mock('../src/utils/model-fetcher'); + jest.mock('../src/utils/api-key-store'); + jest.mock('../src/utils/config'); + jest.mock('../src/utils/logger', () => ({ + logger: { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn() } + })); + + const validator = require('../src/utils/model-validator'); + validateDirectModel = validator.validateDirectModel; + filterRelevantModels = validator.filterRelevantModels; + + const fetcher = require('../src/utils/model-fetcher'); + const keyStore = require('../src/utils/api-key-store'); + const config = require('../src/utils/config'); + + keyStore.readApiKeyValues.mockReturnValue({ google: 'test-key' }); + config.loadConfig.mockReturnValue({ aliases: {} }); + config.saveConfig.mockImplementation(() => {}); + config.getConfigPath.mockReturnValue('/tmp/sidecar-test-config.json'); + fetcher.fetchModelsFromProvider.mockResolvedValue(MOCK_GOOGLE_MODELS); + + // Default to TTY for interactive tests; non-TTY tests override this + process.stdin.isTTY = true; + }); + + describe('validateDirectModel', () => { + it('should return silently when model exists on provider', async () => { + const result = await validateDirectModel('google/gemini-2.5-flash', 'gemini'); + expect(result).toBe('google/gemini-2.5-flash'); + }); + + it('should return model as-is when no provider key is available', async () => { + const keyStore = require('../src/utils/api-key-store'); + keyStore.readApiKeyValues.mockReturnValue({}); + + const result = await validateDirectModel('google/gemini-old', 'gemini'); + expect(result).toBe('google/gemini-old'); + }); + + it('should return model as-is when fetch fails (network error)', async () => { + const fetcher = require('../src/utils/model-fetcher'); + fetcher.fetchModelsFromProvider.mockRejectedValue(new Error('Network error')); + + const result = await validateDirectModel('google/gemini-old', 'gemini'); + expect(result).toBe('google/gemini-old'); + }); + + it('should return model as-is when fetch returns empty list', async () => { + const fetcher = require('../src/utils/model-fetcher'); + fetcher.fetchModelsFromProvider.mockResolvedValue([]); + + const result = await validateDirectModel('google/gemini-old', 'gemini'); + expect(result).toBe('google/gemini-old'); + }); + + it('should return model as-is for malformed model string', async () => { + const result = await validateDirectModel('noSlash', 'gemini'); + expect(result).toBe('noSlash'); + }); + + it('should throw in headless mode when model not found', async () => { + await expect( + validateDirectModel('google/gemini-old-deprecated', 'gemini', { headless: true }) + ).rejects.toThrow(/not found on google API/i); + }); + + it('should include available models in headless error', async () => { + await expect( + validateDirectModel('google/gemini-old-deprecated', 'gemini', { headless: true }) + ).rejects.toThrow(/gemini-2\.5-flash/); + }); + + it('should include fix command in headless error', async () => { + await expect( + validateDirectModel('google/gemini-old-deprecated', 'gemini', { headless: true }) + ).rejects.toThrow(/sidecar setup --add-alias/); + }); + + it('should throw when stdin is not a TTY (non-interactive)', async () => { + process.stdin.isTTY = false; + + await expect( + validateDirectModel('google/gemini-old-deprecated', 'gemini') + ).rejects.toThrow(/not found on google API/i); + }); + + it('should prompt user in interactive mode when model not found', async () => { + // Mock readline to simulate user selecting option 1 + const mockRl = { question: jest.fn(), close: jest.fn() }; + mockRl.question.mockImplementation((_prompt, cb) => cb('1')); + jest.spyOn(require('readline'), 'createInterface').mockReturnValue(mockRl); + + // Suppress stderr output during test + const stderrSpy = jest.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const result = await validateDirectModel('google/gemini-old', 'gemini'); + + // filterRelevantModels sorts by name, so first match is Gemini 1.5 Pro + expect(result).toMatch(/^google\/gemini-/); + expect(mockRl.question).toHaveBeenCalled(); + expect(mockRl.close).toHaveBeenCalled(); + + stderrSpy.mockRestore(); + }); + + it('should save selected model to config', async () => { + const config = require('../src/utils/config'); + const mockRl = { question: jest.fn(), close: jest.fn() }; + mockRl.question.mockImplementation((_prompt, cb) => cb('1')); + jest.spyOn(require('readline'), 'createInterface').mockReturnValue(mockRl); + jest.spyOn(process.stderr, 'write').mockImplementation(() => true); + + await validateDirectModel('google/gemini-old', 'gemini'); + + expect(config.saveConfig).toHaveBeenCalledWith( + expect.objectContaining({ + aliases: expect.objectContaining({ + gemini: expect.stringMatching(/^google\/gemini-/) + }) + }) + ); + + process.stderr.write.mockRestore(); + }); + + it('should throw when config file exists but is malformed', async () => { + const config = require('../src/utils/config'); + const fs = require('fs'); + + config.loadConfig.mockReturnValue(null); + config.getConfigPath.mockReturnValue('/tmp/sidecar-test-config.json'); + jest.spyOn(fs, 'existsSync').mockReturnValue(true); + + const mockRl = { question: jest.fn(), close: jest.fn() }; + mockRl.question.mockImplementation((_prompt, cb) => cb('1')); + jest.spyOn(require('readline'), 'createInterface').mockReturnValue(mockRl); + jest.spyOn(process.stderr, 'write').mockImplementation(() => true); + + await expect( + validateDirectModel('google/gemini-old', 'gemini') + ).rejects.toThrow(/malformed/i); + + expect(config.saveConfig).not.toHaveBeenCalled(); + + process.stderr.write.mockRestore(); + fs.existsSync.mockRestore(); + }); + + it('should create new config when no config file exists', async () => { + const config = require('../src/utils/config'); + const fs = require('fs'); + + config.loadConfig.mockReturnValue(null); + config.getConfigPath.mockReturnValue('/tmp/sidecar-nonexistent-config.json'); + jest.spyOn(fs, 'existsSync').mockReturnValue(false); + + const mockRl = { question: jest.fn(), close: jest.fn() }; + mockRl.question.mockImplementation((_prompt, cb) => cb('1')); + jest.spyOn(require('readline'), 'createInterface').mockReturnValue(mockRl); + jest.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const result = await validateDirectModel('google/gemini-old', 'gemini'); + + expect(result).toMatch(/^google\/gemini-/); + expect(config.saveConfig).toHaveBeenCalled(); + + process.stderr.write.mockRestore(); + fs.existsSync.mockRestore(); + }); + + it('should throw when user cancels (empty input)', async () => { + const mockRl = { question: jest.fn(), close: jest.fn() }; + mockRl.question.mockImplementation((_prompt, cb) => cb('')); + jest.spyOn(require('readline'), 'createInterface').mockReturnValue(mockRl); + jest.spyOn(process.stderr, 'write').mockImplementation(() => true); + + await expect( + validateDirectModel('google/gemini-old', 'gemini') + ).rejects.toThrow(/cancelled/i); + + process.stderr.write.mockRestore(); + }); + + it('should throw when user enters invalid number', async () => { + const mockRl = { question: jest.fn(), close: jest.fn() }; + mockRl.question.mockImplementation((_prompt, cb) => cb('999')); + jest.spyOn(require('readline'), 'createInterface').mockReturnValue(mockRl); + jest.spyOn(process.stderr, 'write').mockImplementation(() => true); + + await expect( + validateDirectModel('google/gemini-old', 'gemini') + ).rejects.toThrow(/cancelled/i); + + process.stderr.write.mockRestore(); + }); + }); + + describe('filterRelevantModels', () => { + it('should filter google models by gemini alias', () => { + const result = filterRelevantModels(MOCK_GOOGLE_MODELS, 'gemini'); + expect(result.every(m => m.id.includes('gemini'))).toBe(true); + expect(result.length).toBe(4); + }); + + it('should filter openai models by gpt alias', () => { + const result = filterRelevantModels(MOCK_OPENAI_MODELS, 'gpt'); + expect(result.every(m => m.id.includes('gpt'))).toBe(true); + expect(result.length).toBe(2); + }); + + it('should return all models when alias has no matches', () => { + const result = filterRelevantModels(MOCK_GOOGLE_MODELS, 'unknownalias'); + expect(result.length).toBe(MOCK_GOOGLE_MODELS.length); + }); + + it('should sort results by name', () => { + const result = filterRelevantModels(MOCK_GOOGLE_MODELS, 'gemini'); + for (let i = 1; i < result.length; i++) { + expect(result[i].name.localeCompare(result[i - 1].name)).toBeGreaterThanOrEqual(0); + } + }); + + it('should limit results to 15', () => { + const manyModels = Array.from({ length: 30 }, (_, i) => ({ + id: `google/gemini-model-${i}`, name: `Gemini Model ${i}` + })); + const result = filterRelevantModels(manyModels, 'gemini'); + expect(result.length).toBe(15); + }); + + it('should use alias search term mapping for opus → claude', () => { + const anthropicModels = [ + { id: 'anthropic/claude-opus-4.6', name: 'Claude Opus 4.6' }, + { id: 'anthropic/claude-sonnet-4.6', name: 'Claude Sonnet 4.6' }, + { id: 'anthropic/some-other-model', name: 'Other Model' }, + ]; + const result = filterRelevantModels(anthropicModels, 'opus'); + expect(result.every(m => m.id.includes('claude'))).toBe(true); + }); + }); +});