diff --git a/packages/types/src/providers/bedrock.ts b/packages/types/src/providers/bedrock.ts index bdffdc146c9..de44e715606 100644 --- a/packages/types/src/providers/bedrock.ts +++ b/packages/types/src/providers/bedrock.ts @@ -19,6 +19,7 @@ export const bedrockModels = { supportsImages: true, supportsPromptCache: true, supportsReasoningBudget: true, + supportsNativeTools: true, inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, @@ -32,6 +33,7 @@ export const bedrockModels = { contextWindow: 300_000, supportsImages: true, supportsPromptCache: true, + supportsNativeTools: true, inputPrice: 0.8, outputPrice: 3.2, cacheWritesPrice: 0.8, // per million tokens @@ -45,6 +47,7 @@ export const bedrockModels = { contextWindow: 300_000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 1.0, outputPrice: 4.0, cacheWritesPrice: 1.0, // per million tokens @@ -56,6 +59,7 @@ export const bedrockModels = { contextWindow: 300_000, supportsImages: true, supportsPromptCache: true, + supportsNativeTools: true, inputPrice: 0.06, outputPrice: 0.24, cacheWritesPrice: 0.06, // per million tokens @@ -69,6 +73,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: false, supportsPromptCache: true, + supportsNativeTools: true, inputPrice: 0.035, outputPrice: 0.14, cacheWritesPrice: 0.035, // per million tokens @@ -83,6 +88,7 @@ export const bedrockModels = { supportsImages: true, supportsPromptCache: true, supportsReasoningBudget: true, + supportsNativeTools: true, inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, @@ -97,6 +103,7 @@ export const bedrockModels = { supportsImages: true, supportsPromptCache: true, supportsReasoningBudget: true, + supportsNativeTools: true, inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, @@ -111,6 +118,7 @@ export const bedrockModels = { supportsImages: true, supportsPromptCache: true, supportsReasoningBudget: true, + supportsNativeTools: true, inputPrice: 5.0, outputPrice: 25.0, cacheWritesPrice: 6.25, @@ -125,6 +133,7 @@ export const bedrockModels = { supportsImages: true, supportsPromptCache: true, supportsReasoningBudget: true, + supportsNativeTools: true, inputPrice: 15.0, outputPrice: 75.0, cacheWritesPrice: 18.75, @@ -139,6 +148,7 @@ export const bedrockModels = { supportsImages: true, supportsPromptCache: true, supportsReasoningBudget: true, + supportsNativeTools: true, inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, @@ -152,6 +162,7 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: true, + supportsNativeTools: true, inputPrice: 3.0, outputPrice: 15.0, cacheWritesPrice: 3.75, @@ -165,6 +176,7 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: false, supportsPromptCache: true, + supportsNativeTools: true, inputPrice: 0.8, outputPrice: 4.0, cacheWritesPrice: 1.0, @@ -179,6 +191,7 @@ export const bedrockModels = { supportsImages: true, supportsPromptCache: true, supportsReasoningBudget: true, + supportsNativeTools: true, inputPrice: 1.0, outputPrice: 5.0, cacheWritesPrice: 1.25, // 5m cache writes @@ -192,6 +205,7 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 3.0, outputPrice: 15.0, }, @@ -200,6 +214,7 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 15.0, outputPrice: 75.0, }, @@ -208,6 +223,7 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 3.0, outputPrice: 15.0, }, @@ -216,6 +232,7 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.25, outputPrice: 1.25, }, @@ -224,6 +241,7 @@ export const bedrockModels = { contextWindow: 100_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 8.0, outputPrice: 24.0, description: "Claude 2.1", @@ -233,6 +251,7 @@ export const bedrockModels = { contextWindow: 100_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 8.0, outputPrice: 24.0, description: "Claude 2.0", @@ -242,6 +261,7 @@ export const bedrockModels = { contextWindow: 100_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.8, outputPrice: 2.4, description: "Claude Instant", @@ -251,6 +271,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 1.35, outputPrice: 5.4, }, @@ -259,6 +280,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.5, outputPrice: 1.5, description: "GPT-OSS 20B - Optimized for low latency and local/specialized use cases", @@ -268,6 +290,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 2.0, outputPrice: 6.0, description: "GPT-OSS 120B - Production-ready, general-purpose, high-reasoning model", @@ -277,6 +300,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.72, outputPrice: 0.72, description: "Llama 3.3 Instruct (70B)", @@ -286,6 +310,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.72, outputPrice: 0.72, description: "Llama 3.2 Instruct (90B)", @@ -295,6 +320,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: true, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.16, outputPrice: 0.16, description: "Llama 3.2 Instruct (11B)", @@ -304,6 +330,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.15, outputPrice: 0.15, description: "Llama 3.2 Instruct (3B)", @@ -313,6 +340,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.1, outputPrice: 0.1, description: "Llama 3.2 Instruct (1B)", @@ -322,6 +350,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 2.4, outputPrice: 2.4, description: "Llama 3.1 Instruct (405B)", @@ -331,6 +360,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.72, outputPrice: 0.72, description: "Llama 3.1 Instruct (70B)", @@ -340,6 +370,7 @@ export const bedrockModels = { contextWindow: 128_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.9, outputPrice: 0.9, description: "Llama 3.1 Instruct (70B) (w/ latency optimized inference)", @@ -349,6 +380,7 @@ export const bedrockModels = { contextWindow: 8_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.22, outputPrice: 0.22, description: "Llama 3.1 Instruct (8B)", @@ -358,6 +390,7 @@ export const bedrockModels = { contextWindow: 8_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 2.65, outputPrice: 3.5, }, @@ -366,6 +399,7 @@ export const bedrockModels = { contextWindow: 4_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.3, outputPrice: 0.6, }, @@ -374,6 +408,7 @@ export const bedrockModels = { contextWindow: 8_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.15, outputPrice: 0.2, description: "Amazon Titan Text Lite", @@ -383,6 +418,7 @@ export const bedrockModels = { contextWindow: 8_000, supportsImages: false, supportsPromptCache: false, + supportsNativeTools: true, inputPrice: 0.2, outputPrice: 0.6, description: "Amazon Titan Text Express", diff --git a/src/api/providers/__tests__/bedrock-native-tools.spec.ts b/src/api/providers/__tests__/bedrock-native-tools.spec.ts new file mode 100644 index 00000000000..e8b13dccc4c --- /dev/null +++ b/src/api/providers/__tests__/bedrock-native-tools.spec.ts @@ -0,0 +1,576 @@ +// Mock AWS SDK credential providers +vi.mock("@aws-sdk/credential-providers", () => { + const mockFromIni = vi.fn().mockReturnValue({ + accessKeyId: "profile-access-key", + secretAccessKey: "profile-secret-key", + }) + return { fromIni: mockFromIni } +}) + +// Mock BedrockRuntimeClient and ConverseStreamCommand +const mockSend = vi.fn() + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + return { + BedrockRuntimeClient: vi.fn().mockImplementation(() => ({ + send: mockSend, + config: { region: "us-east-1" }, + })), + ConverseStreamCommand: vi.fn((params) => ({ + ...params, + input: params, + })), + ConverseCommand: vi.fn(), + } +}) + +import { AwsBedrockHandler } from "../bedrock" +import { ConverseStreamCommand } from "@aws-sdk/client-bedrock-runtime" +import type { ApiHandlerCreateMessageMetadata } from "../../index" + +const mockConverseStreamCommand = vi.mocked(ConverseStreamCommand) + +// Test tool definitions in OpenAI format +const testTools = [ + { + type: "function" as const, + function: { + name: "read_file", + description: "Read a file from the filesystem", + parameters: { + type: "object", + properties: { + path: { type: "string", description: "The path to the file" }, + }, + required: ["path"], + }, + }, + }, + { + type: "function" as const, + function: { + name: "write_file", + description: "Write content to a file", + parameters: { + type: "object", + properties: { + path: { type: "string", description: "The path to the file" }, + content: { type: "string", description: "The content to write" }, + }, + required: ["path", "content"], + }, + }, + }, +] + +describe("AwsBedrockHandler Native Tool Calling", () => { + let handler: AwsBedrockHandler + + beforeEach(() => { + vi.clearAllMocks() + + // Create handler with a model that supports native tools + handler = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + }) + + // Mock the stream response + mockSend.mockResolvedValue({ + stream: [], + }) + }) + + describe("convertToolsForBedrock", () => { + it("should convert OpenAI tools to Bedrock format", () => { + // Access private method + const convertToolsForBedrock = (handler as any).convertToolsForBedrock.bind(handler) + + const bedrockTools = convertToolsForBedrock(testTools) + + expect(bedrockTools).toHaveLength(2) + expect(bedrockTools[0]).toEqual({ + toolSpec: { + name: "read_file", + description: "Read a file from the filesystem", + inputSchema: { + json: { + type: "object", + properties: { + path: { type: "string", description: "The path to the file" }, + }, + required: ["path"], + }, + }, + }, + }) + }) + + it("should filter non-function tools", () => { + const convertToolsForBedrock = (handler as any).convertToolsForBedrock.bind(handler) + + const mixedTools = [ + ...testTools, + { type: "other" as any, something: {} }, // Should be filtered out + ] + + const bedrockTools = convertToolsForBedrock(mixedTools) + + expect(bedrockTools).toHaveLength(2) + }) + }) + + describe("convertToolChoiceForBedrock", () => { + it("should convert 'auto' to Bedrock auto format", () => { + const convertToolChoiceForBedrock = (handler as any).convertToolChoiceForBedrock.bind(handler) + + const result = convertToolChoiceForBedrock("auto") + + expect(result).toEqual({ auto: {} }) + }) + + it("should convert 'required' to Bedrock any format", () => { + const convertToolChoiceForBedrock = (handler as any).convertToolChoiceForBedrock.bind(handler) + + const result = convertToolChoiceForBedrock("required") + + expect(result).toEqual({ any: {} }) + }) + + it("should return undefined for 'none'", () => { + const convertToolChoiceForBedrock = (handler as any).convertToolChoiceForBedrock.bind(handler) + + const result = convertToolChoiceForBedrock("none") + + expect(result).toBeUndefined() + }) + + it("should convert specific tool choice to Bedrock tool format", () => { + const convertToolChoiceForBedrock = (handler as any).convertToolChoiceForBedrock.bind(handler) + + const result = convertToolChoiceForBedrock({ + type: "function", + function: { name: "read_file" }, + }) + + expect(result).toEqual({ + tool: { + name: "read_file", + }, + }) + }) + + it("should default to auto for undefined toolChoice", () => { + const convertToolChoiceForBedrock = (handler as any).convertToolChoiceForBedrock.bind(handler) + + const result = convertToolChoiceForBedrock(undefined) + + expect(result).toEqual({ auto: {} }) + }) + }) + + describe("createMessage with native tools", () => { + it("should include toolConfig when tools are provided with native protocol", async () => { + // Override model info to support native tools + const modelInfo = handler.getModel().info + ;(modelInfo as any).supportsNativeTools = true + + const handlerWithNativeTools = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + }) + + // Manually set supportsNativeTools + const getModelOriginal = handlerWithNativeTools.getModel.bind(handlerWithNativeTools) + handlerWithNativeTools.getModel = () => { + const model = getModelOriginal() + model.info.supportsNativeTools = true + return model + } + + const metadata: ApiHandlerCreateMessageMetadata = { + taskId: "test-task", + tools: testTools, + toolProtocol: "native", + } + + const generator = handlerWithNativeTools.createMessage( + "You are a helpful assistant.", + [{ role: "user", content: "Read the file at /test.txt" }], + metadata, + ) + + await generator.next() + + expect(mockConverseStreamCommand).toHaveBeenCalled() + const commandArg = mockConverseStreamCommand.mock.calls[0][0] as any + + expect(commandArg.toolConfig).toBeDefined() + expect(commandArg.toolConfig.tools).toHaveLength(2) + expect(commandArg.toolConfig.tools[0].toolSpec.name).toBe("read_file") + expect(commandArg.toolConfig.toolChoice).toEqual({ auto: {} }) + }) + + it("should not include toolConfig when toolProtocol is xml", async () => { + const handlerWithNativeTools = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + }) + + // Manually set supportsNativeTools + const getModelOriginal = handlerWithNativeTools.getModel.bind(handlerWithNativeTools) + handlerWithNativeTools.getModel = () => { + const model = getModelOriginal() + model.info.supportsNativeTools = true + return model + } + + const metadata: ApiHandlerCreateMessageMetadata = { + taskId: "test-task", + tools: testTools, + toolProtocol: "xml", // XML protocol should not use native tools + } + + const generator = handlerWithNativeTools.createMessage( + "You are a helpful assistant.", + [{ role: "user", content: "Read the file at /test.txt" }], + metadata, + ) + + await generator.next() + + expect(mockConverseStreamCommand).toHaveBeenCalled() + const commandArg = mockConverseStreamCommand.mock.calls[0][0] as any + + expect(commandArg.toolConfig).toBeUndefined() + }) + + it("should not include toolConfig when tool_choice is none", async () => { + const handlerWithNativeTools = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + }) + + // Manually set supportsNativeTools + const getModelOriginal = handlerWithNativeTools.getModel.bind(handlerWithNativeTools) + handlerWithNativeTools.getModel = () => { + const model = getModelOriginal() + model.info.supportsNativeTools = true + return model + } + + const metadata: ApiHandlerCreateMessageMetadata = { + taskId: "test-task", + tools: testTools, + toolProtocol: "native", + tool_choice: "none", // Explicitly disable tool use + } + + const generator = handlerWithNativeTools.createMessage( + "You are a helpful assistant.", + [{ role: "user", content: "Read the file at /test.txt" }], + metadata, + ) + + await generator.next() + + expect(mockConverseStreamCommand).toHaveBeenCalled() + const commandArg = mockConverseStreamCommand.mock.calls[0][0] as any + + expect(commandArg.toolConfig).toBeUndefined() + }) + + it("should include fine-grained tool streaming beta for Claude models with native tools", async () => { + const handlerWithNativeTools = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + }) + + // Manually set supportsNativeTools + const getModelOriginal = handlerWithNativeTools.getModel.bind(handlerWithNativeTools) + handlerWithNativeTools.getModel = () => { + const model = getModelOriginal() + model.info.supportsNativeTools = true + return model + } + + const metadata: ApiHandlerCreateMessageMetadata = { + taskId: "test-task", + tools: testTools, + toolProtocol: "native", + } + + const generator = handlerWithNativeTools.createMessage( + "You are a helpful assistant.", + [{ role: "user", content: "Read the file at /test.txt" }], + metadata, + ) + + await generator.next() + + expect(mockConverseStreamCommand).toHaveBeenCalled() + const commandArg = mockConverseStreamCommand.mock.calls[0][0] as any + + // Should include the fine-grained tool streaming beta + expect(commandArg.additionalModelRequestFields).toBeDefined() + expect(commandArg.additionalModelRequestFields.anthropic_beta).toContain( + "fine-grained-tool-streaming-2025-05-14", + ) + }) + + it("should not include fine-grained tool streaming beta when not using native tools", async () => { + const handlerWithNativeTools = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + }) + + const metadata: ApiHandlerCreateMessageMetadata = { + taskId: "test-task", + // No tools provided + } + + const generator = handlerWithNativeTools.createMessage( + "You are a helpful assistant.", + [{ role: "user", content: "Hello" }], + metadata, + ) + + await generator.next() + + expect(mockConverseStreamCommand).toHaveBeenCalled() + const commandArg = mockConverseStreamCommand.mock.calls[0][0] as any + + // Should not include anthropic_beta when not using native tools + if (commandArg.additionalModelRequestFields?.anthropic_beta) { + expect(commandArg.additionalModelRequestFields.anthropic_beta).not.toContain( + "fine-grained-tool-streaming-2025-05-14", + ) + } + }) + }) + + describe("tool call streaming events", () => { + it("should yield tool_call_partial for toolUse block start", async () => { + const handlerWithNativeTools = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + }) + + // Mock stream with tool use events + mockSend.mockResolvedValue({ + stream: (async function* () { + yield { + contentBlockStart: { + contentBlockIndex: 0, + start: { + toolUse: { + toolUseId: "tool-123", + name: "read_file", + }, + }, + }, + } + yield { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { + toolUse: { + input: '{"path": "/test.txt"}', + }, + }, + }, + } + yield { + metadata: { + usage: { + inputTokens: 100, + outputTokens: 50, + }, + }, + } + })(), + }) + + const generator = handlerWithNativeTools.createMessage("You are a helpful assistant.", [ + { role: "user", content: "Read the file" }, + ]) + + const results: any[] = [] + for await (const chunk of generator) { + results.push(chunk) + } + + // Should have tool_call_partial chunks + const toolCallChunks = results.filter((r) => r.type === "tool_call_partial") + expect(toolCallChunks).toHaveLength(2) + + // First chunk should have id and name + expect(toolCallChunks[0]).toEqual({ + type: "tool_call_partial", + index: 0, + id: "tool-123", + name: "read_file", + arguments: undefined, + }) + + // Second chunk should have arguments + expect(toolCallChunks[1]).toEqual({ + type: "tool_call_partial", + index: 0, + id: undefined, + name: undefined, + arguments: '{"path": "/test.txt"}', + }) + }) + + it("should yield tool_call_partial for contentBlock toolUse structure", async () => { + const handlerWithNativeTools = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + }) + + // Mock stream with alternative tool use structure + mockSend.mockResolvedValue({ + stream: (async function* () { + yield { + contentBlockStart: { + contentBlockIndex: 0, + contentBlock: { + toolUse: { + toolUseId: "tool-456", + name: "write_file", + }, + }, + }, + } + yield { + metadata: { + usage: { + inputTokens: 100, + outputTokens: 50, + }, + }, + } + })(), + }) + + const generator = handlerWithNativeTools.createMessage("You are a helpful assistant.", [ + { role: "user", content: "Write a file" }, + ]) + + const results: any[] = [] + for await (const chunk of generator) { + results.push(chunk) + } + + // Should have tool_call_partial chunk + const toolCallChunks = results.filter((r) => r.type === "tool_call_partial") + expect(toolCallChunks).toHaveLength(1) + + expect(toolCallChunks[0]).toEqual({ + type: "tool_call_partial", + index: 0, + id: "tool-456", + name: "write_file", + arguments: undefined, + }) + }) + + it("should handle mixed text and tool use content", async () => { + const handlerWithNativeTools = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + }) + + // Mock stream with mixed content + mockSend.mockResolvedValue({ + stream: (async function* () { + yield { + contentBlockStart: { + contentBlockIndex: 0, + start: { + text: "Let me read that file for you.", + }, + }, + } + yield { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { + text: " Here's what I found:", + }, + }, + } + yield { + contentBlockStart: { + contentBlockIndex: 1, + start: { + toolUse: { + toolUseId: "tool-789", + name: "read_file", + }, + }, + }, + } + yield { + contentBlockDelta: { + contentBlockIndex: 1, + delta: { + toolUse: { + input: '{"path": "/example.txt"}', + }, + }, + }, + } + yield { + metadata: { + usage: { + inputTokens: 150, + outputTokens: 75, + }, + }, + } + })(), + }) + + const generator = handlerWithNativeTools.createMessage("You are a helpful assistant.", [ + { role: "user", content: "Read the example file" }, + ]) + + const results: any[] = [] + for await (const chunk of generator) { + results.push(chunk) + } + + // Should have text chunks + const textChunks = results.filter((r) => r.type === "text") + expect(textChunks).toHaveLength(2) + expect(textChunks[0].text).toBe("Let me read that file for you.") + expect(textChunks[1].text).toBe(" Here's what I found:") + + // Should have tool call chunks + const toolCallChunks = results.filter((r) => r.type === "tool_call_partial") + expect(toolCallChunks).toHaveLength(2) + expect(toolCallChunks[0].name).toBe("read_file") + expect(toolCallChunks[1].arguments).toBe('{"path": "/example.txt"}') + }) + }) +}) diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index faaee0360f1..4a4adfc0f41 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -6,7 +6,11 @@ import { ContentBlock, Message, SystemContentBlock, + Tool, + ToolConfiguration, + ToolChoice, } from "@aws-sdk/client-bedrock-runtime" +import OpenAI from "openai" import { fromIni } from "@aws-sdk/credential-providers" import { Anthropic } from "@anthropic-ai/sdk" @@ -67,6 +71,7 @@ interface BedrockPayload { inferenceConfig: BedrockInferenceConfig anthropic_version?: string additionalModelRequestFields?: BedrockAdditionalModelFields + toolConfig?: ToolConfiguration } // Define specific types for content block events to avoid 'as any' usage @@ -75,6 +80,10 @@ interface ContentBlockStartEvent { start?: { text?: string thinking?: string + toolUse?: { + toolUseId?: string + name?: string + } } contentBlockIndex?: number // Alternative structure used by some AWS SDK versions @@ -89,6 +98,11 @@ interface ContentBlockStartEvent { reasoningContent?: { text?: string } + // Tool use block start + toolUse?: { + toolUseId?: string + name?: string + } } } @@ -101,6 +115,10 @@ interface ContentBlockDeltaEvent { reasoningContent?: { text?: string } + // Tool use input delta + toolUse?: { + input?: string + } } contentBlockIndex?: number } @@ -327,6 +345,15 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH const modelConfig = this.getModel() const usePromptCache = Boolean(this.options.awsUsePromptCache && this.supportsAwsPromptCache(modelConfig)) + // Determine early if native tools should be used (needed for message conversion) + const supportsNativeTools = modelConfig.info.supportsNativeTools ?? false + const useNativeTools = + supportsNativeTools && + metadata?.tools && + metadata.tools.length > 0 && + metadata?.toolProtocol !== "xml" && + metadata?.tool_choice !== "none" + const conversationId = messages.length > 0 ? `conv_${messages[0].role}_${ @@ -342,6 +369,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH usePromptCache, modelConfig.info, conversationId, + useNativeTools, ) let additionalModelRequestFields: BedrockAdditionalModelFields | undefined @@ -382,12 +410,36 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH const is1MContextEnabled = BEDROCK_1M_CONTEXT_MODEL_IDS.includes(baseModelId as any) && this.options.awsBedrock1MContext - // Add anthropic_beta for 1M context to additionalModelRequestFields + // Add anthropic_beta headers for various features + // Start with an empty array and add betas as needed + const anthropicBetas: string[] = [] + + // Add 1M context beta if enabled if (is1MContextEnabled) { + anthropicBetas.push("context-1m-2025-08-07") + } + + // Add fine-grained tool streaming beta when native tools are used with Claude models + // This enables proper tool use streaming for Anthropic models on Bedrock + if (useNativeTools && baseModelId.includes("claude")) { + anthropicBetas.push("fine-grained-tool-streaming-2025-05-14") + } + + // Apply anthropic_beta to additionalModelRequestFields if any betas are needed + if (anthropicBetas.length > 0) { if (!additionalModelRequestFields) { additionalModelRequestFields = {} as BedrockAdditionalModelFields } - additionalModelRequestFields.anthropic_beta = ["context-1m-2025-08-07"] + additionalModelRequestFields.anthropic_beta = anthropicBetas + } + + // Build tool configuration if native tools are enabled + let toolConfig: ToolConfiguration | undefined + if (useNativeTools && metadata?.tools) { + toolConfig = { + tools: this.convertToolsForBedrock(metadata.tools), + toolChoice: this.convertToolChoiceForBedrock(metadata.tool_choice), + } } const payload: BedrockPayload = { @@ -398,6 +450,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH ...(additionalModelRequestFields && { additionalModelRequestFields }), // Add anthropic_version at top level when using thinking features ...(thinkingEnabled && { anthropic_version: "bedrock-2023-05-31" }), + ...(toolConfig && { toolConfig }), } // Create AbortController with 10 minute timeout @@ -530,6 +583,19 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH text: contentBlock.thinking, } } + } + // Handle tool use block start + else if (cbStart.start?.toolUse || cbStart.contentBlock?.toolUse) { + const toolUse = cbStart.start?.toolUse || cbStart.contentBlock?.toolUse + if (toolUse) { + yield { + type: "tool_call_partial", + index: cbStart.contentBlockIndex ?? 0, + id: toolUse.toolUseId, + name: toolUse.name, + arguments: undefined, + } + } } else if (cbStart.start?.text) { yield { type: "text", @@ -549,6 +615,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH // - delta.reasoningContent.text: AWS docs structure for reasoning // - delta.thinking: alternative structure for thinking content // - delta.text: standard text content + // - delta.toolUse.input: tool input arguments if (delta) { // Check for reasoningContent property (AWS SDK structure) if (delta.reasoningContent?.text) { @@ -559,6 +626,18 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH continue } + // Handle tool use input delta + if (delta.toolUse?.input) { + yield { + type: "tool_call_partial", + index: cbDelta.contentBlockIndex ?? 0, + id: undefined, + name: undefined, + arguments: delta.toolUse.input, + } + continue + } + // Handle alternative thinking structure (fallback for older SDK versions) if (delta.type === "thinking_delta" && delta.thinking) { yield { @@ -724,9 +803,12 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH usePromptCache: boolean = false, modelInfo?: any, conversationId?: string, // Optional conversation ID to track cache points across messages + useNativeTools: boolean = false, // Whether native tool calling is being used ): { system: SystemContentBlock[]; messages: Message[] } { // First convert messages using shared converter for proper image handling - const convertedMessages = sharedConverter(anthropicMessages as Anthropic.Messages.MessageParam[]) + const convertedMessages = sharedConverter(anthropicMessages as Anthropic.Messages.MessageParam[], { + useNativeTools, + }) // If prompt caching is disabled, return the converted messages directly if (!usePromptCache) { @@ -1054,6 +1136,72 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH return content } + /************************************************************************************ + * + * NATIVE TOOLS + * + *************************************************************************************/ + + /** + * Convert OpenAI tool definitions to Bedrock Converse format + * @param tools Array of OpenAI ChatCompletionTool definitions + * @returns Array of Bedrock Tool definitions + */ + private convertToolsForBedrock(tools: OpenAI.Chat.ChatCompletionTool[]): Tool[] { + return tools + .filter((tool) => tool.type === "function") + .map( + (tool) => + ({ + toolSpec: { + name: tool.function.name, + description: tool.function.description, + inputSchema: { + json: tool.function.parameters as Record, + }, + }, + }) as Tool, + ) + } + + /** + * Convert OpenAI tool_choice to Bedrock ToolChoice format + * @param toolChoice OpenAI tool_choice parameter + * @returns Bedrock ToolChoice configuration + */ + private convertToolChoiceForBedrock( + toolChoice: OpenAI.Chat.ChatCompletionCreateParams["tool_choice"], + ): ToolChoice | undefined { + if (!toolChoice) { + // Default to auto - model decides whether to use tools + return { auto: {} } as ToolChoice + } + + if (typeof toolChoice === "string") { + switch (toolChoice) { + case "none": + return undefined // Bedrock doesn't have "none", just omit tools + case "auto": + return { auto: {} } as ToolChoice + case "required": + return { any: {} } as ToolChoice // Model must use at least one tool + default: + return { auto: {} } as ToolChoice + } + } + + // Handle object form { type: "function", function: { name: string } } + if (typeof toolChoice === "object" && "function" in toolChoice) { + return { + tool: { + name: toolChoice.function.name, + }, + } as ToolChoice + } + + return { auto: {} } as ToolChoice + } + /************************************************************************************ * * AMAZON REGIONS diff --git a/src/api/transform/__tests__/bedrock-converse-format.spec.ts b/src/api/transform/__tests__/bedrock-converse-format.spec.ts index 708aeb17ac3..c0e3e9103d6 100644 --- a/src/api/transform/__tests__/bedrock-converse-format.spec.ts +++ b/src/api/transform/__tests__/bedrock-converse-format.spec.ts @@ -67,7 +67,7 @@ describe("convertToBedrockConverseMessages", () => { } }) - it("converts tool use messages correctly", () => { + it("converts tool use messages correctly (default XML format)", () => { const messages: Anthropic.Messages.MessageParam[] = [ { role: "assistant", @@ -84,6 +84,7 @@ describe("convertToBedrockConverseMessages", () => { }, ] + // Default behavior (useNativeTools: false) converts tool_use to XML text format const result = convertToBedrockConverseMessages(messages) if (!result[0] || !result[0].content) { @@ -91,13 +92,49 @@ describe("convertToBedrockConverseMessages", () => { return } + expect(result[0].role).toBe("assistant") + const textBlock = result[0].content[0] as ContentBlock + if ("text" in textBlock) { + expect(textBlock.text).toContain("") + expect(textBlock.text).toContain("read_file") + expect(textBlock.text).toContain("test.txt") + } else { + expect.fail("Expected text block with XML content not found") + } + }) + + it("converts tool use messages correctly (native tools format)", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "test-id", + name: "read_file", + input: { + path: "test.txt", + }, + }, + ], + }, + ] + + // With useNativeTools: true, keeps tool_use as native format + const result = convertToBedrockConverseMessages(messages, { useNativeTools: true }) + + if (!result[0] || !result[0].content) { + expect.fail("Expected result to have content") + return + } + expect(result[0].role).toBe("assistant") const toolBlock = result[0].content[0] as ContentBlock if ("toolUse" in toolBlock && toolBlock.toolUse) { expect(toolBlock.toolUse).toEqual({ toolUseId: "test-id", name: "read_file", - input: "\n\ntest.txt\n\n", + input: { path: "test.txt" }, }) } else { expect.fail("Expected tool use block not found") @@ -139,6 +176,40 @@ describe("convertToBedrockConverseMessages", () => { } }) + it("converts tool result messages with string content correctly", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "test-id", + content: "File: test.txt\nLines 1-5:\nHello World", + } as any, // Anthropic types don't allow string content but runtime can have it + ], + }, + ] + + const result = convertToBedrockConverseMessages(messages) + + if (!result[0] || !result[0].content) { + expect.fail("Expected result to have content") + return + } + + expect(result[0].role).toBe("user") + const resultBlock = result[0].content[0] as ContentBlock + if ("toolResult" in resultBlock && resultBlock.toolResult) { + expect(resultBlock.toolResult).toEqual({ + toolUseId: "test-id", + content: [{ text: "File: test.txt\nLines 1-5:\nHello World" }], + status: "success", + }) + } else { + expect.fail("Expected tool result block not found") + } + }) + it("handles text content correctly", () => { const messages: Anthropic.Messages.MessageParam[] = [ { diff --git a/src/api/transform/bedrock-converse-format.ts b/src/api/transform/bedrock-converse-format.ts index 1f53067c84e..b6f9b7232a9 100644 --- a/src/api/transform/bedrock-converse-format.ts +++ b/src/api/transform/bedrock-converse-format.ts @@ -24,8 +24,15 @@ interface BedrockMessageContent { /** * Convert Anthropic messages to Bedrock Converse format + * @param anthropicMessages Messages in Anthropic format + * @param options Optional configuration for conversion + * @param options.useNativeTools When true, keeps tool_use input as JSON object instead of XML string */ -export function convertToBedrockConverseMessages(anthropicMessages: Anthropic.Messages.MessageParam[]): Message[] { +export function convertToBedrockConverseMessages( + anthropicMessages: Anthropic.Messages.MessageParam[], + options?: { useNativeTools?: boolean }, +): Message[] { + const useNativeTools = options?.useNativeTools ?? false return anthropicMessages.map((anthropicMessage) => { // Map Anthropic roles to Bedrock roles const role: ConversationRole = anthropicMessage.role === "assistant" ? "assistant" : "user" @@ -46,7 +53,7 @@ export function convertToBedrockConverseMessages(anthropicMessages: Anthropic.Me const messageBlock = block as BedrockMessageContent & { id?: string tool_use_id?: string - content?: Array<{ type: string; text: string }> + content?: string | Array<{ type: string; text: string }> output?: string | Array<{ type: string; text: string }> } @@ -86,32 +93,52 @@ export function convertToBedrockConverseMessages(anthropicMessages: Anthropic.Me } if (messageBlock.type === "tool_use") { - // Convert tool use to XML format - const toolParams = Object.entries(messageBlock.input || {}) - .map(([key, value]) => `<${key}>\n${value}\n`) - .join("\n") - - return { - toolUse: { - toolUseId: messageBlock.id || "", - name: messageBlock.name || "", - input: `<${messageBlock.name}>\n${toolParams}\n`, - }, - } as ContentBlock - } - - if (messageBlock.type === "tool_result") { - // First try to use content if available - if (messageBlock.content && Array.isArray(messageBlock.content)) { + if (useNativeTools) { + // For native tool calling, keep input as JSON object for Bedrock's toolUse format return { - toolResult: { - toolUseId: messageBlock.tool_use_id || "", - content: messageBlock.content.map((item) => ({ - text: item.text, - })), - status: "success", + toolUse: { + toolUseId: messageBlock.id || "", + name: messageBlock.name || "", + input: messageBlock.input || {}, }, } as ContentBlock + } else { + // Convert tool use to XML text format for XML-based tool calling + return { + text: `\n${messageBlock.name}\n${JSON.stringify(messageBlock.input)}\n`, + } as ContentBlock + } + } + + if (messageBlock.type === "tool_result") { + // Handle content field - can be string or array + if (messageBlock.content) { + // Content is a string + if (typeof messageBlock.content === "string") { + return { + toolResult: { + toolUseId: messageBlock.tool_use_id || "", + content: [ + { + text: messageBlock.content, + }, + ], + status: "success", + }, + } as ContentBlock + } + // Content is an array of content blocks + if (Array.isArray(messageBlock.content)) { + return { + toolResult: { + toolUseId: messageBlock.tool_use_id || "", + content: messageBlock.content.map((item) => ({ + text: typeof item === "string" ? item : item.text || String(item), + })), + status: "success", + }, + } as ContentBlock + } } // Fall back to output handling if content is not available