diff --git a/packages/types/src/providers/lite-llm.ts b/packages/types/src/providers/lite-llm.ts index 14a68cfc3c3..2121cce4f88 100644 --- a/packages/types/src/providers/lite-llm.ts +++ b/packages/types/src/providers/lite-llm.ts @@ -8,6 +8,7 @@ export const litellmDefaultModelInfo: ModelInfo = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + supportsNativeTools: true, inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, diff --git a/src/api/providers/fetchers/__tests__/litellm.spec.ts b/src/api/providers/fetchers/__tests__/litellm.spec.ts index a93c21ee1b0..bf36f3e456e 100644 --- a/src/api/providers/fetchers/__tests__/litellm.spec.ts +++ b/src/api/providers/fetchers/__tests__/litellm.spec.ts @@ -222,8 +222,11 @@ describe("getLiteLLMModels", () => { contextWindow: 200000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: 3, outputPrice: 15, + cacheWritesPrice: undefined, + cacheReadsPrice: undefined, description: "claude-3-5-sonnet via LiteLLM proxy", }, "gpt-4-turbo": { @@ -231,8 +234,11 @@ describe("getLiteLLMModels", () => { contextWindow: 128000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: 10, outputPrice: 30, + cacheWritesPrice: undefined, + cacheReadsPrice: undefined, description: "gpt-4-turbo via LiteLLM proxy", }, }) @@ -299,8 +305,11 @@ describe("getLiteLLMModels", () => { contextWindow: 200000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: undefined, outputPrice: undefined, + cacheWritesPrice: undefined, + cacheReadsPrice: undefined, description: "test-computer-model via LiteLLM proxy", }) @@ -309,8 +318,11 @@ describe("getLiteLLMModels", () => { contextWindow: 200000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: undefined, outputPrice: undefined, + cacheWritesPrice: undefined, + cacheReadsPrice: undefined, description: "test-non-computer-model via LiteLLM proxy", }) }) @@ -443,8 +455,11 @@ describe("getLiteLLMModels", () => { contextWindow: 200000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: undefined, outputPrice: undefined, + cacheWritesPrice: undefined, + cacheReadsPrice: undefined, description: "claude-3-5-sonnet-latest via LiteLLM proxy", }) @@ -453,8 +468,11 @@ describe("getLiteLLMModels", () => { contextWindow: 128000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: undefined, outputPrice: undefined, + cacheWritesPrice: undefined, + cacheReadsPrice: undefined, description: "gpt-4-turbo via LiteLLM proxy", }) }) @@ -515,8 +533,11 @@ describe("getLiteLLMModels", () => { contextWindow: 200000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: undefined, outputPrice: undefined, + cacheWritesPrice: undefined, + cacheReadsPrice: undefined, description: "claude-3-5-sonnet-latest via LiteLLM proxy", }) @@ -525,8 +546,11 @@ describe("getLiteLLMModels", () => { contextWindow: 128000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: undefined, outputPrice: undefined, + cacheWritesPrice: undefined, + cacheReadsPrice: undefined, description: "custom-model via LiteLLM proxy", }) @@ -535,8 +559,11 @@ describe("getLiteLLMModels", () => { contextWindow: 128000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: undefined, outputPrice: undefined, + cacheWritesPrice: undefined, + cacheReadsPrice: undefined, description: "another-custom-model via LiteLLM proxy", }) }) @@ -646,6 +673,7 @@ describe("getLiteLLMModels", () => { contextWindow: 200000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: undefined, outputPrice: undefined, cacheWritesPrice: undefined, @@ -659,6 +687,7 @@ describe("getLiteLLMModels", () => { contextWindow: 128000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: undefined, outputPrice: undefined, cacheWritesPrice: undefined, @@ -672,6 +701,7 @@ describe("getLiteLLMModels", () => { contextWindow: 100000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: false, inputPrice: undefined, outputPrice: undefined, cacheWritesPrice: undefined, diff --git a/src/api/providers/fetchers/litellm.ts b/src/api/providers/fetchers/litellm.ts index e6c96f3c2d1..f1a3a1ae935 100644 --- a/src/api/providers/fetchers/litellm.ts +++ b/src/api/providers/fetchers/litellm.ts @@ -45,6 +45,11 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise contextWindow: modelInfo.max_input_tokens || 200000, supportsImages: Boolean(modelInfo.supports_vision), supportsPromptCache: Boolean(modelInfo.supports_prompt_caching), + supportsNativeTools: Boolean( + modelInfo.supports_function_calling || + modelInfo.supports_tool_choice || + modelInfo.supports_tool_use, + ), inputPrice: modelInfo.input_cost_per_token ? modelInfo.input_cost_per_token * 1000000 : undefined, outputPrice: modelInfo.output_cost_per_token ? modelInfo.output_cost_per_token * 1000000 diff --git a/src/api/providers/lite-llm.ts b/src/api/providers/lite-llm.ts index 43bf33c38be..9acbc35d074 100644 --- a/src/api/providers/lite-llm.ts +++ b/src/api/providers/lite-llm.ts @@ -1,7 +1,7 @@ import OpenAI from "openai" import { Anthropic } from "@anthropic-ai/sdk" // Keep for type usage only -import { litellmDefaultModelId, litellmDefaultModelInfo } from "@roo-code/types" +import { litellmDefaultModelId, litellmDefaultModelInfo, TOOL_PROTOCOL } from "@roo-code/types" import { calculateApiCostOpenAI } from "../../shared/cost" @@ -116,6 +116,14 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa // Check if this is a GPT-5 model that requires max_completion_tokens instead of max_tokens const isGPT5Model = this.isGpt5(modelId) + // Check if model supports native tools and tools are provided with native protocol + const supportsNativeTools = info.supportsNativeTools ?? false + const useNativeTools = + supportsNativeTools && + metadata?.tools && + metadata.tools.length > 0 && + metadata?.toolProtocol === TOOL_PROTOCOL.NATIVE + const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { model: modelId, messages: [systemMessage, ...enhancedMessages], @@ -123,6 +131,9 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa stream_options: { include_usage: true, }, + ...(useNativeTools && { tools: this.convertToolsForOpenAI(metadata.tools) }), + ...(useNativeTools && metadata.tool_choice && { tool_choice: metadata.tool_choice }), + ...(useNativeTools && { parallel_tool_calls: metadata?.parallelToolCalls ?? false }), } // GPT-5 models require max_completion_tokens instead of the deprecated max_tokens parameter @@ -149,6 +160,19 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa yield { type: "text", text: delta.content } } + // Handle tool calls in stream - emit partial chunks for NativeToolCallParser + if (delta?.tool_calls) { + for (const toolCall of delta.tool_calls) { + yield { + type: "tool_call_partial", + index: toolCall.index, + id: toolCall.id, + name: toolCall.function?.name, + arguments: toolCall.function?.arguments, + } + } + } + if (usage) { lastUsage = usage }