Skip to content

feat: add native tool support for LiteLLM provider #9719

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mrubens merged 2 commits into from
Dec 1, 2025
Merged

feat: add native tool support for LiteLLM provider #9719

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e877088
feat: add native tool support for LiteLLM provider
daniel-lxs Dec 1, 2025
17e0e57
fix: require explicit toolProtocol === NATIVE for native tools activa…
daniel-lxs Dec 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/types/src/providers/lite-llm.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export const litellmDefaultModelInfo: ModelInfo = {
contextWindow: 200_000,
supportsImages: true,
supportsPromptCache: true,
supportsNativeTools: true,
inputPrice: 3.0,
outputPrice: 15.0,
cacheWritesPrice: 3.75,
Expand Down
30 changes: 30 additions & 0 deletions src/api/providers/fetchers/__tests__/litellm.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,17 +222,23 @@ describe("getLiteLLMModels", () => {
contextWindow: 200000,
supportsImages: true,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: 3,
outputPrice: 15,
cacheWritesPrice: undefined,
cacheReadsPrice: undefined,
description: "claude-3-5-sonnet via LiteLLM proxy",
},
"gpt-4-turbo": {
maxTokens: 8192,
contextWindow: 128000,
supportsImages: false,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: 10,
outputPrice: 30,
cacheWritesPrice: undefined,
cacheReadsPrice: undefined,
description: "gpt-4-turbo via LiteLLM proxy",
},
})
Expand Down Expand Up @@ -299,8 +305,11 @@ describe("getLiteLLMModels", () => {
contextWindow: 200000,
supportsImages: true,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: undefined,
outputPrice: undefined,
cacheWritesPrice: undefined,
cacheReadsPrice: undefined,
description: "test-computer-model via LiteLLM proxy",
})

Expand All @@ -309,8 +318,11 @@ describe("getLiteLLMModels", () => {
contextWindow: 200000,
supportsImages: false,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: undefined,
outputPrice: undefined,
cacheWritesPrice: undefined,
cacheReadsPrice: undefined,
description: "test-non-computer-model via LiteLLM proxy",
})
})
Expand Down Expand Up @@ -443,8 +455,11 @@ describe("getLiteLLMModels", () => {
contextWindow: 200000,
supportsImages: true,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: undefined,
outputPrice: undefined,
cacheWritesPrice: undefined,
cacheReadsPrice: undefined,
description: "claude-3-5-sonnet-latest via LiteLLM proxy",
})

Expand All @@ -453,8 +468,11 @@ describe("getLiteLLMModels", () => {
contextWindow: 128000,
supportsImages: false,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: undefined,
outputPrice: undefined,
cacheWritesPrice: undefined,
cacheReadsPrice: undefined,
description: "gpt-4-turbo via LiteLLM proxy",
})
})
Expand Down Expand Up @@ -515,8 +533,11 @@ describe("getLiteLLMModels", () => {
contextWindow: 200000,
supportsImages: true,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: undefined,
outputPrice: undefined,
cacheWritesPrice: undefined,
cacheReadsPrice: undefined,
description: "claude-3-5-sonnet-latest via LiteLLM proxy",
})

Expand All @@ -525,8 +546,11 @@ describe("getLiteLLMModels", () => {
contextWindow: 128000,
supportsImages: false,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: undefined,
outputPrice: undefined,
cacheWritesPrice: undefined,
cacheReadsPrice: undefined,
description: "custom-model via LiteLLM proxy",
})

Expand All @@ -535,8 +559,11 @@ describe("getLiteLLMModels", () => {
contextWindow: 128000,
supportsImages: false,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: undefined,
outputPrice: undefined,
cacheWritesPrice: undefined,
cacheReadsPrice: undefined,
description: "another-custom-model via LiteLLM proxy",
})
})
Expand Down Expand Up @@ -646,6 +673,7 @@ describe("getLiteLLMModels", () => {
contextWindow: 200000,
supportsImages: true,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: undefined,
outputPrice: undefined,
cacheWritesPrice: undefined,
Expand All @@ -659,6 +687,7 @@ describe("getLiteLLMModels", () => {
contextWindow: 128000,
supportsImages: false,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: undefined,
outputPrice: undefined,
cacheWritesPrice: undefined,
Expand All @@ -672,6 +701,7 @@ describe("getLiteLLMModels", () => {
contextWindow: 100000,
supportsImages: false,
supportsPromptCache: false,
supportsNativeTools: false,
inputPrice: undefined,
outputPrice: undefined,
cacheWritesPrice: undefined,
Expand Down
5 changes: 5 additions & 0 deletions src/api/providers/fetchers/litellm.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise
contextWindow: modelInfo.max_input_tokens || 200000,
supportsImages: Boolean(modelInfo.supports_vision),
supportsPromptCache: Boolean(modelInfo.supports_prompt_caching),
supportsNativeTools: Boolean(
modelInfo.supports_function_calling ||
modelInfo.supports_tool_choice ||
modelInfo.supports_tool_use,
),
inputPrice: modelInfo.input_cost_per_token ? modelInfo.input_cost_per_token * 1000000 : undefined,
outputPrice: modelInfo.output_cost_per_token
? modelInfo.output_cost_per_token * 1000000
Expand Down
26 changes: 25 additions & 1 deletion src/api/providers/lite-llm.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import OpenAI from "openai"
import { Anthropic } from "@anthropic-ai/sdk" // Keep for type usage only

import { litellmDefaultModelId, litellmDefaultModelInfo } from "@roo-code/types"
import { litellmDefaultModelId, litellmDefaultModelInfo, TOOL_PROTOCOL } from "@roo-code/types"

import { calculateApiCostOpenAI } from "../../shared/cost"

Expand Down Expand Up @@ -116,13 +116,24 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa
// Check if this is a GPT-5 model that requires max_completion_tokens instead of max_tokens
const isGPT5Model = this.isGpt5(modelId)

// Check if model supports native tools and tools are provided with native protocol
const supportsNativeTools = info.supportsNativeTools ?? false
const useNativeTools =
supportsNativeTools &&
metadata?.tools &&
metadata.tools.length > 0 &&
metadata?.toolProtocol === TOOL_PROTOCOL.NATIVE

const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {
model: modelId,
messages: [systemMessage, ...enhancedMessages],
stream: true,
stream_options: {
include_usage: true,
},
...(useNativeTools && { tools: this.convertToolsForOpenAI(metadata.tools) }),
...(useNativeTools && metadata.tool_choice && { tool_choice: metadata.tool_choice }),
...(useNativeTools && { parallel_tool_calls: metadata?.parallelToolCalls ?? false }),
}

// GPT-5 models require max_completion_tokens instead of the deprecated max_tokens parameter
Expand All @@ -149,6 +160,19 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa
yield { type: "text", text: delta.content }
}

// Handle tool calls in stream - emit partial chunks for NativeToolCallParser
if (delta?.tool_calls) {
for (const toolCall of delta.tool_calls) {
yield {
type: "tool_call_partial",
index: toolCall.index,
id: toolCall.id,
name: toolCall.function?.name,
arguments: toolCall.function?.arguments,
}
}
}

if (usage) {
lastUsage = usage
}
Expand Down
Loading