From c7f319d94b59c20a98418ef90b105bd02ac14da8 Mon Sep 17 00:00:00 2001 From: Daniel Riccio Date: Sat, 10 May 2025 01:57:20 -0500 Subject: [PATCH 01/11] feat: add a new property to get provider specific information about a model --- src/api/providers/openrouter.ts | 41 +++++++++++++++++++++++++-------- src/schemas/index.ts | 2 ++ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 1eec3930210..f6a609410ca 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -174,24 +174,47 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH } override getModel() { - const id = this.options.openRouterModelId ?? openRouterDefaultModelId - const info = this.models[id] ?? openRouterDefaultModelInfo + const modelId = this.options.openRouterModelId ?? openRouterDefaultModelId + + let baseModelInfo = this.models[modelId] ?? openRouterDefaultModelInfo + let finalModelInfo = baseModelInfo + + if ( + this.options.openRouterSelectedProviderInfo && + this.options.openRouterSpecificProvider && + this.options.openRouterSpecificProvider !== OPENROUTER_DEFAULT_PROVIDER_NAME && + baseModelInfo + ) { + finalModelInfo = { + ...baseModelInfo, + contextWindow: this.options.openRouterSelectedProviderInfo.contextWindow, + maxTokens: this.options.openRouterSelectedProviderInfo.maxTokens ?? baseModelInfo.maxTokens, + inputPrice: this.options.openRouterSelectedProviderInfo.inputPrice ?? baseModelInfo.inputPrice, + outputPrice: this.options.openRouterSelectedProviderInfo.outputPrice ?? baseModelInfo.outputPrice, + supportsImages: + this.options.openRouterSelectedProviderInfo.supportsImages ?? baseModelInfo.supportsImages, + supportsPromptCache: + this.options.openRouterSelectedProviderInfo.supportsPromptCache ?? + baseModelInfo.supportsPromptCache, + } + } else if (!baseModelInfo) { + finalModelInfo = openRouterDefaultModelInfo + } - const isDeepSeekR1 = id.startsWith("deepseek/deepseek-r1") || id === "perplexity/sonar-reasoning" + const isDeepSeekR1 = modelId.startsWith("deepseek/deepseek-r1") || modelId === "perplexity/sonar-reasoning" return { - id, - info, - // maxTokens, thinking, temperature, reasoningEffort + id: modelId, + info: finalModelInfo, ...getModelParams({ options: this.options, - model: info, + model: finalModelInfo, defaultTemperature: isDeepSeekR1 ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0, }), topP: isDeepSeekR1 ? 0.95 : undefined, promptCache: { - supported: PROMPT_CACHING_MODELS.has(id), - optional: OPTIONAL_PROMPT_CACHING_MODELS.has(id), + supported: finalModelInfo.supportsPromptCache && PROMPT_CACHING_MODELS.has(modelId), + optional: finalModelInfo.supportsPromptCache && OPTIONAL_PROMPT_CACHING_MODELS.has(modelId), }, } } diff --git a/src/schemas/index.ts b/src/schemas/index.ts index 094b11b10d0..a188eeb0aae 100644 --- a/src/schemas/index.ts +++ b/src/schemas/index.ts @@ -360,6 +360,7 @@ export const providerSettingsSchema = z.object({ openRouterModelId: z.string().optional(), openRouterBaseUrl: z.string().optional(), openRouterSpecificProvider: z.string().optional(), + openRouterSelectedProviderInfo: modelInfoSchema.optional(), openRouterUseMiddleOutTransform: z.boolean().optional(), // Amazon Bedrock awsAccessKey: z.string().optional(), @@ -468,6 +469,7 @@ const providerSettingsRecord: ProviderSettingsRecord = { openRouterModelId: undefined, openRouterBaseUrl: undefined, openRouterSpecificProvider: undefined, + openRouterSelectedProviderInfo: undefined, openRouterUseMiddleOutTransform: undefined, // Amazon Bedrock awsAccessKey: undefined, From 725b617162468b27a842c258921d993e9b4c2179 Mon Sep 17 00:00:00 2001 From: Daniel Riccio Date: Sat, 10 May 2025 01:59:18 -0500 Subject: [PATCH 02/11] feat: set the provider specific info for a model --- .../settings/providers/OpenRouter.tsx | 5 +++- .../components/ui/hooks/useSelectedModel.ts | 28 ++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/webview-ui/src/components/settings/providers/OpenRouter.tsx b/webview-ui/src/components/settings/providers/OpenRouter.tsx index 696fcca3316..c8d0997dad4 100644 --- a/webview-ui/src/components/settings/providers/OpenRouter.tsx +++ b/webview-ui/src/components/settings/providers/OpenRouter.tsx @@ -144,7 +144,10 @@ export const OpenRouter = ({ { - setApiConfigurationField("openRouterSpecificProvider", value) - setApiConfigurationField("openRouterSelectedProviderInfo", openRouterModelProviders[value]) - }}> + onValueChange={(value) => setApiConfigurationField("openRouterSpecificProvider", value)}> diff --git a/webview-ui/src/components/ui/hooks/useOpenRouterModelProviders.ts b/webview-ui/src/components/ui/hooks/useOpenRouterModelProviders.ts index 8a881acd59f..41938248d53 100644 --- a/webview-ui/src/components/ui/hooks/useOpenRouterModelProviders.ts +++ b/webview-ui/src/components/ui/hooks/useOpenRouterModelProviders.ts @@ -34,7 +34,7 @@ const openRouterEndpointsSchema = z.object({ }), }) -type OpenRouterModelProvider = ModelInfo & { +export type OpenRouterModelProvider = ModelInfo & { label: string } diff --git a/webview-ui/src/components/ui/hooks/useSelectedModel.ts b/webview-ui/src/components/ui/hooks/useSelectedModel.ts index 0804dcadcb0..694cdce044c 100644 --- a/webview-ui/src/components/ui/hooks/useSelectedModel.ts +++ b/webview-ui/src/components/ui/hooks/useSelectedModel.ts @@ -34,54 +34,64 @@ import { } from "@roo/shared/api" import { useRouterModels } from "./useRouterModels" +import { useOpenRouterModelProviders } from "./useOpenRouterModelProviders" export const useSelectedModel = (apiConfiguration?: ProviderSettings) => { - const { data: routerModels, isLoading, isError } = useRouterModels() const provider = apiConfiguration?.apiProvider || "anthropic" + const openRouterModelId = provider === "openrouter" ? apiConfiguration?.openRouterModelId : undefined + + const routerModels = useRouterModels() + const openRouterModelProviders = useOpenRouterModelProviders(openRouterModelId) const { id, info } = - apiConfiguration && routerModels - ? getSelectedModel({ provider, apiConfiguration, routerModels }) + apiConfiguration && + typeof routerModels.data !== "undefined" && + typeof openRouterModelProviders.data !== "undefined" + ? getSelectedModel({ + provider, + apiConfiguration, + routerModels: routerModels.data, + openRouterModelProviders: openRouterModelProviders.data, + }) : { id: anthropicDefaultModelId, info: undefined } - return { provider, id, info, isLoading, isError } + return { + provider, + id, + info, + isLoading: routerModels.isLoading || openRouterModelProviders.isLoading, + isError: routerModels.isError || openRouterModelProviders.isError, + } } function getSelectedModel({ provider, apiConfiguration, routerModels, + openRouterModelProviders, }: { provider: ProviderName apiConfiguration: ProviderSettings routerModels: RouterModels + openRouterModelProviders: Record }): { id: string; info: ModelInfo } { switch (provider) { case "openrouter": { - const modelId = apiConfiguration.openRouterModelId ?? openRouterDefaultModelId - let baseModelInfo = routerModels.openrouter[modelId] - let finalModelInfo = baseModelInfo + const id = apiConfiguration.openRouterModelId ?? openRouterDefaultModelId + let info = routerModels.openrouter[id] + const specificProvider = apiConfiguration.openRouterSpecificProvider - if (apiConfiguration.openRouterSelectedProviderInfo && baseModelInfo) { - finalModelInfo = { - ...baseModelInfo, - contextWindow: apiConfiguration.openRouterSelectedProviderInfo.contextWindow, - maxTokens: apiConfiguration.openRouterSelectedProviderInfo.maxTokens ?? baseModelInfo.maxTokens, - inputPrice: apiConfiguration.openRouterSelectedProviderInfo.inputPrice ?? baseModelInfo.inputPrice, - outputPrice: - apiConfiguration.openRouterSelectedProviderInfo.outputPrice ?? baseModelInfo.outputPrice, - supportsImages: - apiConfiguration.openRouterSelectedProviderInfo.supportsImages ?? baseModelInfo.supportsImages, - supportsPromptCache: - apiConfiguration.openRouterSelectedProviderInfo.supportsPromptCache ?? - baseModelInfo.supportsPromptCache, - } - } else if (!baseModelInfo) { - finalModelInfo = routerModels.openrouter[openRouterDefaultModelId] + if (specificProvider && openRouterModelProviders[specificProvider]) { + console.log( + `openRouterSpecificProvider: ${specificProvider} ->`, + openRouterModelProviders[specificProvider], + ) + + info = openRouterModelProviders[specificProvider] } - return finalModelInfo - ? { id: modelId, info: finalModelInfo } + return info + ? { id, info } : { id: openRouterDefaultModelId, info: routerModels.openrouter[openRouterDefaultModelId] } } case "requesty": { From 736b97ae0c38fb71311a6426bf61753611c2ae2f Mon Sep 17 00:00:00 2001 From: cte Date: Mon, 12 May 2025 15:37:18 -0700 Subject: [PATCH 06/11] Revert this --- .../src/components/ui/hooks/useOpenRouterModelProviders.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webview-ui/src/components/ui/hooks/useOpenRouterModelProviders.ts b/webview-ui/src/components/ui/hooks/useOpenRouterModelProviders.ts index 41938248d53..8a881acd59f 100644 --- a/webview-ui/src/components/ui/hooks/useOpenRouterModelProviders.ts +++ b/webview-ui/src/components/ui/hooks/useOpenRouterModelProviders.ts @@ -34,7 +34,7 @@ const openRouterEndpointsSchema = z.object({ }), }) -export type OpenRouterModelProvider = ModelInfo & { +type OpenRouterModelProvider = ModelInfo & { label: string } From f93419a58de5b659fa95eb1fbf7d4cef47effda8 Mon Sep 17 00:00:00 2001 From: cte Date: Mon, 12 May 2025 16:10:36 -0700 Subject: [PATCH 07/11] Add caching layer for OpenRouter provider --- package-lock.json | 25 +++++++ package.json | 1 + .../fetchers/{cache.ts => modelCache.ts} | 9 ++- .../providers/fetchers/modelEndpointCache.ts | 74 +++++++++++++++++++ src/api/providers/openrouter.ts | 21 +++++- src/api/providers/requesty.ts | 2 +- src/api/providers/router-provider.ts | 2 +- src/core/webview/webviewMessageHandler.ts | 2 +- 8 files changed, 128 insertions(+), 8 deletions(-) rename src/api/providers/fetchers/{cache.ts => modelCache.ts} (94%) create mode 100644 src/api/providers/fetchers/modelEndpointCache.ts diff --git a/package-lock.json b/package-lock.json index 27d5492eb47..6080636b6fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,6 +51,7 @@ "puppeteer-chromium-resolver": "^23.0.0", "puppeteer-core": "^23.4.0", "reconnecting-eventsource": "^1.6.4", + "sanitize-filename": "^1.6.3", "say": "^0.16.0", "serialize-error": "^11.0.3", "simple-git": "^3.27.0", @@ -18264,6 +18265,15 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/sanitize-filename": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.3.tgz", + "integrity": "sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==", + "license": "WTFPL OR ISC", + "dependencies": { + "truncate-utf8-bytes": "^1.0.0" + } + }, "node_modules/sax": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", @@ -19483,6 +19493,15 @@ "tree-sitter-wasms": "^0.1.11" } }, + "node_modules/truncate-utf8-bytes": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", + "integrity": "sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==", + "license": "WTFPL", + "dependencies": { + "utf8-byte-length": "^1.0.1" + } + }, "node_modules/ts-api-utils": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz", @@ -19930,6 +19949,12 @@ "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz", "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==" }, + "node_modules/utf8-byte-length": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.5.tgz", + "integrity": "sha512-Xn0w3MtiQ6zoz2vFyUVruaCL53O/DwUvkEeOvj+uulMm0BkUGYWmBYVyElqZaSLhY6ZD0ulfU3aBra2aVT4xfA==", + "license": "(WTFPL OR MIT)" + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index a125e3b32c7..ce0c1b75dbe 100644 --- a/package.json +++ b/package.json @@ -407,6 +407,7 @@ "puppeteer-chromium-resolver": "^23.0.0", "puppeteer-core": "^23.4.0", "reconnecting-eventsource": "^1.6.4", + "sanitize-filename": "^1.6.3", "say": "^0.16.0", "serialize-error": "^11.0.3", "simple-git": "^3.27.0", diff --git a/src/api/providers/fetchers/cache.ts b/src/api/providers/fetchers/modelCache.ts similarity index 94% rename from src/api/providers/fetchers/cache.ts rename to src/api/providers/fetchers/modelCache.ts index aefc3961a50..9ab4b851fc5 100644 --- a/src/api/providers/fetchers/cache.ts +++ b/src/api/providers/fetchers/modelCache.ts @@ -47,6 +47,7 @@ export const getModels = async ( baseUrl: string | undefined = undefined, ): Promise => { let models = memoryCache.get(router) + if (models) { // console.log(`[getModels] NodeCache hit for ${router} -> ${Object.keys(models).length}`) return models @@ -82,7 +83,9 @@ export const getModels = async ( try { await writeModels(router, models) // console.log(`[getModels] wrote ${router} models to file cache`) - } catch (error) {} + } catch (error) { + console.error(`[getModels] error writing ${router} models to file cache`, error) + } return models } @@ -90,7 +93,9 @@ export const getModels = async ( try { models = await readModels(router) // console.log(`[getModels] read ${router} models from file cache`) - } catch (error) {} + } catch (error) { + console.error(`[getModels] error reading ${router} models from file cache`, error) + } return models ?? {} } diff --git a/src/api/providers/fetchers/modelEndpointCache.ts b/src/api/providers/fetchers/modelEndpointCache.ts new file mode 100644 index 00000000000..488e70ab7ff --- /dev/null +++ b/src/api/providers/fetchers/modelEndpointCache.ts @@ -0,0 +1,74 @@ +import * as path from "path" +import fs from "fs/promises" + +import NodeCache from "node-cache" +import sanitize from "sanitize-filename" + +import { ContextProxy } from "../../../core/config/ContextProxy" +import { getCacheDirectoryPath } from "../../../shared/storagePathManager" +import { RouterName, ModelRecord } from "../../../shared/api" +import { fileExistsAtPath } from "../../../utils/fs" + +import { getOpenRouterModelEndpoints } from "./openrouter" + +const memoryCache = new NodeCache({ stdTTL: 5 * 60, checkperiod: 5 * 60 }) + +const getCacheKey = (router: RouterName, modelId: string) => sanitize(`${router}_${modelId}`) + +async function writeModelEndpoints(key: string, data: ModelRecord) { + const filename = `${key}_endpoints.json` + const cacheDir = await getCacheDirectoryPath(ContextProxy.instance.globalStorageUri.fsPath) + await fs.writeFile(path.join(cacheDir, filename), JSON.stringify(data, null, 2)) +} + +async function readModelEndpoints(key: string): Promise { + const filename = `${key}_endpoints.json` + const cacheDir = await getCacheDirectoryPath(ContextProxy.instance.globalStorageUri.fsPath) + const filePath = path.join(cacheDir, filename) + const exists = await fileExistsAtPath(filePath) + return exists ? JSON.parse(await fs.readFile(filePath, "utf8")) : undefined +} + +export const getModelEndpoints = async (router: RouterName, modelId?: string): Promise => { + // OpenRouter is the only provider that supports model endpoints, but you + // can see how we'd extend this to other providers in the future. + if (router !== "openrouter" || !modelId) { + return {} + } + + const key = getCacheKey(router, modelId) + let modelProviders = memoryCache.get(key) + + if (modelProviders) { + // console.log(`[getModelProviders] NodeCache hit for ${key} -> ${Object.keys(modelProviders).length}`) + return modelProviders + } + + modelProviders = await getOpenRouterModelEndpoints(modelId) + + if (Object.keys(modelProviders).length > 0) { + // console.log(`[getModelProviders] API fetch for ${key} -> ${Object.keys(modelProviders).length}`) + memoryCache.set(key, modelProviders) + + try { + await writeModelEndpoints(key, modelProviders) + // console.log(`[getModelProviders] wrote ${key} endpoints to file cache`) + } catch (error) { + console.error(`[getModelProviders] error writing ${key} endpoints to file cache`, error) + } + + return modelProviders + } + + try { + modelProviders = await readModelEndpoints(router) + // console.log(`[getModelProviders] read ${key} endpoints from file cache`) + } catch (error) { + console.error(`[getModelProviders] error reading ${key} endpoints from file cache`, error) + } + + return modelProviders ?? {} +} + +export const flushModelProviders = async (router: RouterName, modelId: string) => + memoryCache.del(getCacheKey(router, modelId)) diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 39694c8baaa..8887787b56a 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -20,7 +20,8 @@ import { addCacheBreakpoints as addGeminiCacheBreakpoints } from "../transform/c import { getModelParams, SingleCompletionHandler } from "../index" import { DEFAULT_HEADERS, DEEP_SEEK_DEFAULT_TEMPERATURE } from "./constants" import { BaseProvider } from "./base-provider" -import { getModels } from "./fetchers/cache" +import { getModels } from "./fetchers/modelCache" +import { getModelEndpoints } from "./fetchers/modelEndpointCache" const OPENROUTER_DEFAULT_PROVIDER_NAME = "[default]" @@ -57,6 +58,7 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH protected options: ApiHandlerOptions private client: OpenAI protected models: ModelRecord = {} + protected endpoints: ModelRecord = {} constructor(options: ApiHandlerOptions) { super() @@ -168,13 +170,26 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH } public async fetchModel() { - this.models = await getModels("openrouter") + const [models, endpoints] = await Promise.all([ + getModels("openrouter"), + getModelEndpoints("openrouter", this.options.openRouterModelId), + ]) + + this.models = models + this.endpoints = endpoints + return this.getModel() } override getModel() { const id = this.options.openRouterModelId ?? openRouterDefaultModelId - const info = this.models[id] ?? openRouterDefaultModelInfo + let info = this.models[id] ?? openRouterDefaultModelInfo + + // If a specific provider is requested, use the endpoint for that provider. + if (this.options.openRouterSpecificProvider && this.endpoints[this.options.openRouterSpecificProvider]) { + info = this.endpoints[this.options.openRouterSpecificProvider] + console.log(`[OpenRouterHandler] Using specific provider: ${this.options.openRouterSpecificProvider}`, info) + } const isDeepSeekR1 = id.startsWith("deepseek/deepseek-r1") || id === "perplexity/sonar-reasoning" diff --git a/src/api/providers/requesty.ts b/src/api/providers/requesty.ts index a8a18bd4ddd..05ca1e67f18 100644 --- a/src/api/providers/requesty.ts +++ b/src/api/providers/requesty.ts @@ -12,7 +12,7 @@ import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { SingleCompletionHandler } from "../" import { BaseProvider } from "./base-provider" import { DEFAULT_HEADERS } from "./constants" -import { getModels } from "./fetchers/cache" +import { getModels } from "./fetchers/modelCache" import OpenAI from "openai" // Requesty usage includes an extra field for Anthropic use cases. diff --git a/src/api/providers/router-provider.ts b/src/api/providers/router-provider.ts index cd30ae03736..a0decdcab41 100644 --- a/src/api/providers/router-provider.ts +++ b/src/api/providers/router-provider.ts @@ -2,7 +2,7 @@ import OpenAI from "openai" import { ApiHandlerOptions, RouterName, ModelRecord, ModelInfo } from "../../shared/api" import { BaseProvider } from "./base-provider" -import { getModels } from "./fetchers/cache" +import { getModels } from "./fetchers/modelCache" type RouterProviderOptions = { name: RouterName diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index cde453d4fc7..1f17d221a62 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -34,7 +34,7 @@ import { TelemetrySetting } from "../../shared/TelemetrySetting" import { getWorkspacePath } from "../../utils/path" import { Mode, defaultModeSlug } from "../../shared/modes" import { GlobalState } from "../../schemas" -import { getModels, flushModels } from "../../api/providers/fetchers/cache" +import { getModels, flushModels } from "../../api/providers/fetchers/modelCache" import { generateSystemPrompt } from "./generateSystemPrompt" const ALLOWED_VSCODE_SETTINGS = new Set(["terminal.integrated.inheritEnv"]) From b43977c8eb901cd5f1a3820c95c994303401b1a1 Mon Sep 17 00:00:00 2001 From: cte Date: Mon, 12 May 2025 16:15:06 -0700 Subject: [PATCH 08/11] Fix tests --- src/api/providers/__tests__/glama.test.ts | 2 +- src/api/providers/__tests__/openrouter.test.ts | 2 +- src/api/providers/__tests__/requesty.test.ts | 2 +- src/api/providers/__tests__/unbound.test.ts | 2 +- src/api/providers/fetchers/modelEndpointCache.ts | 12 ++++++++++-- src/api/providers/openrouter.ts | 6 +++++- 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/api/providers/__tests__/glama.test.ts b/src/api/providers/__tests__/glama.test.ts index c44debddff0..58d9b6ab670 100644 --- a/src/api/providers/__tests__/glama.test.ts +++ b/src/api/providers/__tests__/glama.test.ts @@ -6,7 +6,7 @@ import { GlamaHandler } from "../glama" import { ApiHandlerOptions } from "../../../shared/api" // Mock dependencies -jest.mock("../fetchers/cache", () => ({ +jest.mock("../fetchers/modelCache", () => ({ getModels: jest.fn().mockImplementation(() => { return Promise.resolve({ "anthropic/claude-3-7-sonnet": { diff --git a/src/api/providers/__tests__/openrouter.test.ts b/src/api/providers/__tests__/openrouter.test.ts index 7b2998f8e7f..5ae7f607a93 100644 --- a/src/api/providers/__tests__/openrouter.test.ts +++ b/src/api/providers/__tests__/openrouter.test.ts @@ -9,7 +9,7 @@ import { ApiHandlerOptions } from "../../../shared/api" // Mock dependencies jest.mock("openai") jest.mock("delay", () => jest.fn(() => Promise.resolve())) -jest.mock("../fetchers/cache", () => ({ +jest.mock("../fetchers/modelCache", () => ({ getModels: jest.fn().mockImplementation(() => { return Promise.resolve({ "anthropic/claude-3.7-sonnet": { diff --git a/src/api/providers/__tests__/requesty.test.ts b/src/api/providers/__tests__/requesty.test.ts index 3e6803c4aa3..b999a01947e 100644 --- a/src/api/providers/__tests__/requesty.test.ts +++ b/src/api/providers/__tests__/requesty.test.ts @@ -8,7 +8,7 @@ import { ApiHandlerOptions } from "../../../shared/api" jest.mock("openai") jest.mock("delay", () => jest.fn(() => Promise.resolve())) -jest.mock("../fetchers/cache", () => ({ +jest.mock("../fetchers/modelCache", () => ({ getModels: jest.fn().mockImplementation(() => { return Promise.resolve({ "coding/claude-3-7-sonnet": { diff --git a/src/api/providers/__tests__/unbound.test.ts b/src/api/providers/__tests__/unbound.test.ts index 3ceacf4d2e5..c01a2b53676 100644 --- a/src/api/providers/__tests__/unbound.test.ts +++ b/src/api/providers/__tests__/unbound.test.ts @@ -7,7 +7,7 @@ import { ApiHandlerOptions } from "../../../shared/api" import { UnboundHandler } from "../unbound" // Mock dependencies -jest.mock("../fetchers/cache", () => ({ +jest.mock("../fetchers/modelCache", () => ({ getModels: jest.fn().mockImplementation(() => { return Promise.resolve({ "anthropic/claude-3-5-sonnet-20241022": { diff --git a/src/api/providers/fetchers/modelEndpointCache.ts b/src/api/providers/fetchers/modelEndpointCache.ts index 488e70ab7ff..23c486e4204 100644 --- a/src/api/providers/fetchers/modelEndpointCache.ts +++ b/src/api/providers/fetchers/modelEndpointCache.ts @@ -29,10 +29,18 @@ async function readModelEndpoints(key: string): Promise return exists ? JSON.parse(await fs.readFile(filePath, "utf8")) : undefined } -export const getModelEndpoints = async (router: RouterName, modelId?: string): Promise => { +export const getModelEndpoints = async ({ + router, + modelId, + endpoint, +}: { + router: RouterName + modelId?: string + endpoint?: string +}): Promise => { // OpenRouter is the only provider that supports model endpoints, but you // can see how we'd extend this to other providers in the future. - if (router !== "openrouter" || !modelId) { + if (router !== "openrouter" || !modelId || !endpoint) { return {} } diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 8887787b56a..9982603baa3 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -172,7 +172,11 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH public async fetchModel() { const [models, endpoints] = await Promise.all([ getModels("openrouter"), - getModelEndpoints("openrouter", this.options.openRouterModelId), + getModelEndpoints({ + router: "openrouter", + modelId: this.options.openRouterModelId, + endpoint: this.options.openRouterSpecificProvider, + }), ]) this.models = models From 6f63d7e0abf1e29767adc7d6caf427a1e18a2441 Mon Sep 17 00:00:00 2001 From: cte Date: Mon, 12 May 2025 16:15:58 -0700 Subject: [PATCH 09/11] Revert this --- webview-ui/src/utils/model-utils.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/webview-ui/src/utils/model-utils.ts b/webview-ui/src/utils/model-utils.ts index 638216ba05f..29adb11a963 100644 --- a/webview-ui/src/utils/model-utils.ts +++ b/webview-ui/src/utils/model-utils.ts @@ -100,9 +100,7 @@ export const calculateTokenDistribution = ( // Get the actual max tokens value from the model // If maxTokens is valid, use it, otherwise reserve 20% of the context window as a default - const defaultReserved = Math.ceil(safeContextWindow * 0.2) - const reservedForOutput = - maxTokens && maxTokens > 0 && maxTokens === safeContextWindow ? defaultReserved : (maxTokens ?? defaultReserved) + const reservedForOutput = maxTokens && maxTokens > 0 ? maxTokens : Math.ceil(safeContextWindow * 0.2) // Calculate sizes directly without buffer display const availableSize = Math.max(0, safeContextWindow - safeContextTokens - reservedForOutput) From af517b391ad2c7d8e2e68f7db20232de3d59280d Mon Sep 17 00:00:00 2001 From: cte Date: Mon, 12 May 2025 16:16:19 -0700 Subject: [PATCH 10/11] Remove logging --- webview-ui/src/components/ui/hooks/useSelectedModel.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/webview-ui/src/components/ui/hooks/useSelectedModel.ts b/webview-ui/src/components/ui/hooks/useSelectedModel.ts index 694cdce044c..0bfa136b39b 100644 --- a/webview-ui/src/components/ui/hooks/useSelectedModel.ts +++ b/webview-ui/src/components/ui/hooks/useSelectedModel.ts @@ -82,11 +82,6 @@ function getSelectedModel({ const specificProvider = apiConfiguration.openRouterSpecificProvider if (specificProvider && openRouterModelProviders[specificProvider]) { - console.log( - `openRouterSpecificProvider: ${specificProvider} ->`, - openRouterModelProviders[specificProvider], - ) - info = openRouterModelProviders[specificProvider] } From 4c6043c118d2dadf93c0e00a9a2c2539af320653 Mon Sep 17 00:00:00 2001 From: cte Date: Mon, 12 May 2025 16:17:12 -0700 Subject: [PATCH 11/11] Remove logging --- src/api/providers/openrouter.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 9982603baa3..2d9c7f8b8a4 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -192,7 +192,6 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH // If a specific provider is requested, use the endpoint for that provider. if (this.options.openRouterSpecificProvider && this.endpoints[this.options.openRouterSpecificProvider]) { info = this.endpoints[this.options.openRouterSpecificProvider] - console.log(`[OpenRouterHandler] Using specific provider: ${this.options.openRouterSpecificProvider}`, info) } const isDeepSeekR1 = id.startsWith("deepseek/deepseek-r1") || id === "perplexity/sonar-reasoning"