From bf7beba5de8c1f7d2bfcfecb413c40163e7c27f3 Mon Sep 17 00:00:00 2001 From: Und3rf10w Date: Fri, 16 May 2025 17:37:32 -0400 Subject: [PATCH 01/10] feat(examples/google): Added google and vertex generate-text examples for reasoning and codeExecution --- .../generate-text/google-code-execution.ts | 44 +++++++++++++++++++ .../src/generate-text/google-reasoning.ts | 9 +++- .../google-vertex-code-execution.ts | 43 ++++++++++++++++++ .../generate-text/google-vertex-reasoning.ts | 26 +++++++++++ 4 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 examples/ai-core/src/generate-text/google-code-execution.ts create mode 100644 examples/ai-core/src/generate-text/google-vertex-code-execution.ts create mode 100644 examples/ai-core/src/generate-text/google-vertex-reasoning.ts diff --git a/examples/ai-core/src/generate-text/google-code-execution.ts b/examples/ai-core/src/generate-text/google-code-execution.ts new file mode 100644 index 000000000000..6a2fbd4c83c3 --- /dev/null +++ b/examples/ai-core/src/generate-text/google-code-execution.ts @@ -0,0 +1,44 @@ + +import { google } from '@ai-sdk/google'; +import { generateText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = await generateText({ + model: google('gemini-2.5-flash-preview-04-17'), + providerOptions: { + google: { + useCodeExecution: true, + } + }, + maxOutputTokens: 2048, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); + + const parts = (result.response?.body as any)?.candidates?.[0]?.content?.parts; + + if (parts && Array.isArray(parts)) { + parts.forEach((part, index) => { + if ('text' in part) { + console.log('\nType: Text'); + console.log('Content:', part.text); + } else if ('executableCode' in part && part.executableCode) { + console.log('\nType: Executable Code'); + console.log('Language:', part.executableCode.language); + console.log('Code:\n', part.executableCode.code); + } else if ('codeExecutionResult' in part && part.codeExecutionResult) { + console.log('\nType: Code Execution Result'); + console.log('Outcome:', part.codeExecutionResult.outcome); + console.log('Output:\n', part.codeExecutionResult.output); + } else { + console.log('\nType: Unknown'); + console.log(JSON.stringify(part, null, 2)); + } + }); + } else { + console.warn('Could not find parts'); + } +} + +main().catch(console.error); \ No newline at end of file diff --git a/examples/ai-core/src/generate-text/google-reasoning.ts b/examples/ai-core/src/generate-text/google-reasoning.ts index f813c67f875b..fca4e91b4b1b 100644 --- a/examples/ai-core/src/generate-text/google-reasoning.ts +++ b/examples/ai-core/src/generate-text/google-reasoning.ts @@ -4,7 +4,14 @@ import 'dotenv/config'; async function main() { const result = await generateText({ - model: google('gemini-2.5-pro-exp-03-25'), + model: google('gemini-2.5-pro-preview-03-25'), + providerOptions: { + google: { + thinkingConfig: { + thinkingBudget: 1024, + }, + }, + }, prompt: 'How many "r"s are in the word "strawberry"?', }); diff --git a/examples/ai-core/src/generate-text/google-vertex-code-execution.ts b/examples/ai-core/src/generate-text/google-vertex-code-execution.ts new file mode 100644 index 000000000000..b388df6137f2 --- /dev/null +++ b/examples/ai-core/src/generate-text/google-vertex-code-execution.ts @@ -0,0 +1,43 @@ +import { vertex } from '@ai-sdk/google-vertex'; +import { generateText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = await generateText({ + model: vertex('gemini-2.5-pro-preview-05-06'), + providerOptions: { + google: { + useCodeExecution: true, + } + }, + maxOutputTokens: 2048, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); + + const parts = (result.response?.body as any)?.candidates?.[0]?.content?.parts; + + if (parts && Array.isArray(parts)) { + parts.forEach((part, index) => { + if ('text' in part) { + console.log('\nType: Text'); + console.log('Content:', part.text); + } else if ('executableCode' in part && part.executableCode) { + console.log('\nType: Executable Code'); + console.log('Language:', part.executableCode.language); + console.log('Code:\n', part.executableCode.code); + } else if ('codeExecutionResult' in part && part.codeExecutionResult) { + console.log('\nType: Code Execution Result'); + console.log('Outcome:', part.codeExecutionResult.outcome); + console.log('Output:\n', part.codeExecutionResult.output); + } else { + console.log('\nType: Unknown'); + console.log(JSON.stringify(part, null, 2)); + } + }); + } else { + console.warn('Could not find parts'); + } +} + +main().catch(console.error); \ No newline at end of file diff --git a/examples/ai-core/src/generate-text/google-vertex-reasoning.ts b/examples/ai-core/src/generate-text/google-vertex-reasoning.ts new file mode 100644 index 000000000000..169d5df25778 --- /dev/null +++ b/examples/ai-core/src/generate-text/google-vertex-reasoning.ts @@ -0,0 +1,26 @@ +import { vertex } from '@ai-sdk/google-vertex'; +import { generateText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = await generateText({ + model: vertex('gemini-2.5-flash-preview-04-17'), + providerOptions: { + google: { + thinkingConfig: { + thinkingBudget: 1024, + includeThoughts: true + }, + }, + }, + prompt: 'How many "r"s are in the word "strawberry"?', + }); + + process.stdout.write('\x1b[34m' + result.reasoningText + '\x1b[0m'); + console.log(result.text); + console.log(); + console.log('Token usage:', result.usage); + console.log('Finish reason:', result.finishReason); +} + +main().catch(console.error); From 2bfb9115065e8e5ffb35e53fb1998c6e4bafb67e Mon Sep 17 00:00:00 2001 From: Und3rf10w Date: Fri, 16 May 2025 19:00:05 -0400 Subject: [PATCH 02/10] feat(examples/google): Tweak reasoning and codeExecution generate-text examples to support new return types --- .../generate-text/google-code-execution.ts | 43 ++++++++++--------- .../src/generate-text/google-reasoning.ts | 1 + .../google-vertex-code-execution.ts | 43 ++++++++++--------- .../generate-text/google-vertex-reasoning.ts | 1 + 4 files changed, 46 insertions(+), 42 deletions(-) diff --git a/examples/ai-core/src/generate-text/google-code-execution.ts b/examples/ai-core/src/generate-text/google-code-execution.ts index 6a2fbd4c83c3..d89959b93aa8 100644 --- a/examples/ai-core/src/generate-text/google-code-execution.ts +++ b/examples/ai-core/src/generate-text/google-code-execution.ts @@ -16,28 +16,29 @@ async function main() { 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', }); - const parts = (result.response?.body as any)?.candidates?.[0]?.content?.parts; - - if (parts && Array.isArray(parts)) { - parts.forEach((part, index) => { - if ('text' in part) { - console.log('\nType: Text'); - console.log('Content:', part.text); - } else if ('executableCode' in part && part.executableCode) { - console.log('\nType: Executable Code'); - console.log('Language:', part.executableCode.language); - console.log('Code:\n', part.executableCode.code); - } else if ('codeExecutionResult' in part && part.codeExecutionResult) { - console.log('\nType: Code Execution Result'); - console.log('Outcome:', part.codeExecutionResult.outcome); - console.log('Output:\n', part.codeExecutionResult.output); - } else { - console.log('\nType: Unknown'); - console.log(JSON.stringify(part, null, 2)); + for (const part of result.content) { + switch (part.type) { + case 'file': { + if (part.type === 'file') { + process.stdout.write( + '\x1b[33m' + + part.type + + '\x1b[34m: ' + + part.file.mediaType + + '\x1b[0m' + ); + console.log(); + console.log(atob(part.file.base64)) + } + } + case 'text': { + if (part.type === 'text') { + process.stdout.write('\x1b[34m' + part.type + '\x1b[0m'); + console.log(); + console.log(part.text) + } } - }); - } else { - console.warn('Could not find parts'); + } } } diff --git a/examples/ai-core/src/generate-text/google-reasoning.ts b/examples/ai-core/src/generate-text/google-reasoning.ts index fca4e91b4b1b..86538355615a 100644 --- a/examples/ai-core/src/generate-text/google-reasoning.ts +++ b/examples/ai-core/src/generate-text/google-reasoning.ts @@ -12,6 +12,7 @@ async function main() { }, }, }, + maxOutputTokens: 2048, prompt: 'How many "r"s are in the word "strawberry"?', }); diff --git a/examples/ai-core/src/generate-text/google-vertex-code-execution.ts b/examples/ai-core/src/generate-text/google-vertex-code-execution.ts index b388df6137f2..6bbba812d8f9 100644 --- a/examples/ai-core/src/generate-text/google-vertex-code-execution.ts +++ b/examples/ai-core/src/generate-text/google-vertex-code-execution.ts @@ -15,28 +15,29 @@ async function main() { 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', }); - const parts = (result.response?.body as any)?.candidates?.[0]?.content?.parts; - - if (parts && Array.isArray(parts)) { - parts.forEach((part, index) => { - if ('text' in part) { - console.log('\nType: Text'); - console.log('Content:', part.text); - } else if ('executableCode' in part && part.executableCode) { - console.log('\nType: Executable Code'); - console.log('Language:', part.executableCode.language); - console.log('Code:\n', part.executableCode.code); - } else if ('codeExecutionResult' in part && part.codeExecutionResult) { - console.log('\nType: Code Execution Result'); - console.log('Outcome:', part.codeExecutionResult.outcome); - console.log('Output:\n', part.codeExecutionResult.output); - } else { - console.log('\nType: Unknown'); - console.log(JSON.stringify(part, null, 2)); + for (const part of result.content) { + switch (part.type) { + case 'file': { + if (part.type === 'file') { + process.stdout.write( + '\x1b[33m' + + part.type + + '\x1b[34m: ' + + part.file.mediaType + + '\x1b[0m' + ); + console.log(); + console.log(atob(part.file.base64)) + } + } + case 'text': { + if (part.type === 'text') { + process.stdout.write('\x1b[34m' + part.type + '\x1b[0m'); + console.log(); + console.log(part.text) + } } - }); - } else { - console.warn('Could not find parts'); + } } } diff --git a/examples/ai-core/src/generate-text/google-vertex-reasoning.ts b/examples/ai-core/src/generate-text/google-vertex-reasoning.ts index 169d5df25778..3b6b568a0b2b 100644 --- a/examples/ai-core/src/generate-text/google-vertex-reasoning.ts +++ b/examples/ai-core/src/generate-text/google-vertex-reasoning.ts @@ -13,6 +13,7 @@ async function main() { }, }, }, + maxOutputTokens: 2048, prompt: 'How many "r"s are in the word "strawberry"?', }); From 2b01e7061b11317e9ac0ecbada1608593db041eb Mon Sep 17 00:00:00 2001 From: Und3rf10w Date: Fri, 16 May 2025 19:30:03 -0400 Subject: [PATCH 03/10] feat(examples/google): Added google and vertex stream-text examples for reasoning and codeExecution --- .../generate-text/google-code-execution.ts | 9 +++ .../google-vertex-code-execution.ts | 9 +++ .../src/stream-text/google-code-execution.ts | 55 +++++++++++++++++++ .../google-vertex-code-execution.ts | 54 ++++++++++++++++++ .../stream-text/google-vertex-reasoning.ts | 35 ++++++++++++ 5 files changed, 162 insertions(+) create mode 100644 examples/ai-core/src/stream-text/google-code-execution.ts create mode 100644 examples/ai-core/src/stream-text/google-vertex-code-execution.ts create mode 100644 examples/ai-core/src/stream-text/google-vertex-reasoning.ts diff --git a/examples/ai-core/src/generate-text/google-code-execution.ts b/examples/ai-core/src/generate-text/google-code-execution.ts index d89959b93aa8..e4adea90afda 100644 --- a/examples/ai-core/src/generate-text/google-code-execution.ts +++ b/examples/ai-core/src/generate-text/google-code-execution.ts @@ -40,6 +40,15 @@ async function main() { } } } + + process.stdout.write('\n\n'); + + console.log(); + console.log('Warnings:', await result.warnings); + + console.log(); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); } main().catch(console.error); \ No newline at end of file diff --git a/examples/ai-core/src/generate-text/google-vertex-code-execution.ts b/examples/ai-core/src/generate-text/google-vertex-code-execution.ts index 6bbba812d8f9..63e10a937126 100644 --- a/examples/ai-core/src/generate-text/google-vertex-code-execution.ts +++ b/examples/ai-core/src/generate-text/google-vertex-code-execution.ts @@ -39,6 +39,15 @@ async function main() { } } } + + process.stdout.write('\n\n'); + + console.log(); + console.log('Warnings:', await result.warnings); + + console.log(); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); } main().catch(console.error); \ No newline at end of file diff --git a/examples/ai-core/src/stream-text/google-code-execution.ts b/examples/ai-core/src/stream-text/google-code-execution.ts new file mode 100644 index 000000000000..b27a816fc260 --- /dev/null +++ b/examples/ai-core/src/stream-text/google-code-execution.ts @@ -0,0 +1,55 @@ + +import { google } from '@ai-sdk/google'; +import { streamText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = streamText({ + model: google('gemini-2.5-flash-preview-04-17'), + maxOutputTokens: 10000, + providerOptions: { + google: { + useCodeExecution: true + } + }, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); + + let fullResponse = ''; + + for await (const delta of result.fullStream) { + switch (delta.type) { + case 'file': { + if (delta.type === 'file') { + process.stdout.write( + '\x1b[33m' + + delta.type + + '\x1b[34m: ' + + delta.file.mediaType + + '\x1b[0m' + ); + console.log(); + console.log(atob(delta.file.base64 as string)); + } + } + case 'text': { + if (delta.type === 'text') { + process.stdout.write('\x1b[34m' + delta.type + '\x1b[0m'); + console.log(); + console.log(delta.text); + fullResponse += delta.text; + } + break; + } + } + } + console.log(); + console.log('Warnings:', await result.warnings); + + console.log(); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); +} + +main().catch(console.log); \ No newline at end of file diff --git a/examples/ai-core/src/stream-text/google-vertex-code-execution.ts b/examples/ai-core/src/stream-text/google-vertex-code-execution.ts new file mode 100644 index 000000000000..f8b1bfbfd211 --- /dev/null +++ b/examples/ai-core/src/stream-text/google-vertex-code-execution.ts @@ -0,0 +1,54 @@ +import { vertex } from '@ai-sdk/google-vertex'; +import { streamText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = streamText({ + model: vertex('gemini-2.5-flash-preview-04-17'), + providerOptions: { + google: { + useCodeExecution: true + } + }, + maxOutputTokens: 10000, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); + + let fullResponse = ''; + + for await (const delta of result.fullStream) { + switch (delta.type) { + case 'file': { + if (delta.type === 'file') { + process.stdout.write( + '\x1b[33m' + + delta.type + + '\x1b[34m: ' + + delta.file.mediaType + + '\x1b[0m' + ); + console.log(); + console.log(atob(delta.file.base64 as string)); + } + } + case 'text': { + if (delta.type === 'text') { + process.stdout.write('\x1b[34m' + delta.type + '\x1b[0m'); + console.log(); + console.log(delta.text); + fullResponse += delta.text; + } + break; + } + } + } + console.log(); + console.log('Warnings:', await result.warnings); + + console.log(); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); +} + +main().catch(console.log); \ No newline at end of file diff --git a/examples/ai-core/src/stream-text/google-vertex-reasoning.ts b/examples/ai-core/src/stream-text/google-vertex-reasoning.ts new file mode 100644 index 000000000000..61455643d641 --- /dev/null +++ b/examples/ai-core/src/stream-text/google-vertex-reasoning.ts @@ -0,0 +1,35 @@ +import { vertex } from '@ai-sdk/google-vertex'; +import { streamText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = streamText({ + model: vertex('gemini-2.5-flash-preview-04-17'), + prompt: 'Tell me the history of the San Francisco Mission-style burrito.', + providerOptions: { + google: { + thinkingConfig: { + thinkingBudget: 1024, + includeThoughts: true + }, + }, + }, + }); + + for await (const part of result.fullStream) { + if (part.type === 'reasoning') { + process.stdout.write('\x1b[34m' + part.text + '\x1b[0m'); + } else if (part.type === 'text') { + process.stdout.write(part.text); + } + } + + console.log(); + console.log('Warnings:', await result.warnings); + + console.log(); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); +} + +main().catch(console.error); From 73d1d35537f77c56de5c6ab6fdcc0677ce125005 Mon Sep 17 00:00:00 2001 From: Und3rf10w Date: Fri, 16 May 2025 19:30:31 -0400 Subject: [PATCH 04/10] feat(providers/google): Implement reasoning and code execution support --- .../google-generative-ai-language-model.ts | 185 ++++++++++++++++-- .../src/google-generative-ai-options.ts | 20 ++ packages/google/src/google-prepare-tools.ts | 87 +++++--- 3 files changed, 245 insertions(+), 47 deletions(-) diff --git a/packages/google/src/google-generative-ai-language-model.ts b/packages/google/src/google-generative-ai-language-model.ts index 3a7bf1ea3810..7733d8d8a3e3 100644 --- a/packages/google/src/google-generative-ai-language-model.ts +++ b/packages/google/src/google-generative-ai-language-model.ts @@ -91,6 +91,20 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { schema: googleGenerativeAIProviderOptions, }); + if ( + googleOptions?.thinkingConfig?.includeThoughts === true && + !this.config.provider.startsWith('google.vertex.') + ) { + warnings.push({ + type: 'other', + message: + "The 'includeThoughts' option is only supported with the Google Vertex provider " + + 'and might not be supported or could behave unexpectedly with the current Google provider ' + + `(${this.config.provider}).`, + }); + } + + const { contents, systemInstruction } = convertToGoogleGenerativeAIMessages(prompt); @@ -104,8 +118,11 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { useSearchGrounding: googleOptions?.useSearchGrounding ?? false, dynamicRetrievalConfig: googleOptions?.dynamicRetrievalConfig, modelId: this.modelId, + useCodeExecution: googleOptions?.useCodeExecution ?? false, }); + warnings.push(...toolWarnings); + return { args: { generationConfig: { @@ -124,11 +141,11 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { responseFormat?.type === 'json' ? 'application/json' : undefined, responseSchema: responseFormat?.type === 'json' && - responseFormat.schema != null && - // Google GenAI does not support all OpenAPI Schema features, - // so this is needed as an escape hatch: - // TODO convert into provider option - (googleOptions?.structuredOutputs ?? true) + responseFormat.schema != null && + // Google GenAI does not support all OpenAPI Schema features, + // so this is needed as an escape hatch: + // TODO convert into provider option + (googleOptions?.structuredOutputs ?? true) ? convertJSONSchemaToOpenAPISchema(responseFormat.schema) : undefined, ...(googleOptions?.audioTimestamp && { @@ -183,14 +200,43 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { // map ordered parts to content: const parts = candidate.content == null || - typeof candidate.content !== 'object' || - !('parts' in candidate.content) + typeof candidate.content !== 'object' || + !('parts' in candidate.content) ? [] : (candidate.content.parts ?? []); for (const part of parts) { - if ('text' in part && part.text.length > 0) { + // Text parts + if ('text' in part && !part.thought && part.text.length > 0) { content.push({ type: 'text', text: part.text }); + // Reasoning parts + } else if ( + 'text' in part && + (part as any).thought === true && + !('executableCode' in part) && + !('codeExecutionResult' in part) && + part.text != null && + part.text.length > 0 + ) { + content.push({ type: 'reasoning', text: part.text }); + // code exectuion: Executable code + } else if ('executableCode' in part && part.executableCode != null && part.executableCode.code.length > 0) { + /** + * NOTE: The vertex api just returns a string, but the ai-sdk expects either a base64 string or uint8Arry. + * So we just convert it to base64 + */ + content.push({ + type: 'file', + mediaType: 'text/x-python', + data: Buffer.from(part.executableCode.code, 'utf-8').toString('base64') + }) + // code execution: Execution result + } else if ('codeExecutionResult' in part && part.codeExecutionResult != null && part.codeExecutionResult.outcome.length > 0) { + content.push({ + type: 'text' as const, + text: `Code Execution Result (Outcome: ${part.codeExecutionResult.outcome}):\n ${part.codeExecutionResult.output}`, + }) + // function calls } else if ('functionCall' in part) { content.push({ type: 'tool-call' as const, @@ -199,6 +245,7 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { toolName: part.functionCall.name, args: JSON.stringify(part.functionCall.args), }); + // inline data } else if ('inlineData' in part) { content.push({ type: 'file' as const, @@ -325,9 +372,25 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { // Process tool call's parts before determining finishReason to ensure hasToolCalls is properly set if (content != null) { - const deltaText = getTextFromParts(content.parts); - if (deltaText != null) { - controller.enqueue(deltaText); + const reasoningContent = getReasoningParts(content.parts); + if (reasoningContent?.type === 'reasoning' && reasoningContent.text.length > 0) { + controller.enqueue({ type: 'reasoning', text: reasoningContent.text }); + } + + // Process code execution parts prior to text parts + const executableCodeFilePart = getExecutableCodeFilePartFromStreamParts(content.parts); + if (executableCodeFilePart != null) { + controller.enqueue(executableCodeFilePart); + } + + const codeExecutionResultTextParts = getCodeExecutionResultStreamParts(content.parts); + for (const textPart of codeExecutionResultTextParts) { + controller.enqueue(textPart); + } + + const textContent = getTextFromParts(content.parts); // getTextFromParts is now refactored + if (textContent?.text && textContent.text.length > 0) { + controller.enqueue({ type: 'text', text: textContent.text }); } const inlineDataParts = getInlineDataParts(content.parts); @@ -428,25 +491,58 @@ function getToolCallsFromParts({ return functionCallParts == null || functionCallParts.length === 0 ? undefined : functionCallParts.map(part => ({ - type: 'tool-call' as const, - toolCallType: 'function' as const, - toolCallId: generateId(), - toolName: part.functionCall.name, - args: JSON.stringify(part.functionCall.args), - })); + type: 'tool-call' as const, + toolCallType: 'function' as const, + toolCallId: generateId(), + toolName: part.functionCall.name, + args: JSON.stringify(part.functionCall.args), + })); } function getTextFromParts(parts: z.infer['parts']) { - const textParts = parts?.filter(part => 'text' in part) as Array< + // Only include plain text parts (not thoughts, not executable code, not code execution results) + const textParts = parts?.filter(part => 'text' in part && part.text != null && !(part as { thought?: boolean }).thought) as Array< GoogleGenerativeAIContentPart & { text: string } >; return textParts == null || textParts.length === 0 ? undefined : { - type: 'text' as const, - text: textParts.map(part => part.text).join(''), - }; + type: 'text' as const, + text: textParts.map(part => part.text).join(''), + }; +} + +function getExecutableCodeFilePartFromStreamParts( + parts: z.infer['parts'], +): LanguageModelV2StreamPart | undefined { + const part = parts?.find(p => 'executableCode' in p && p.executableCode != null); + if (part && 'executableCode' in part && part.executableCode && part.executableCode.code.length > 0) { + return { + type: 'file', + mediaType: 'text/x-python', + data: Buffer.from(part.executableCode.code, 'utf-8').toString('base64'), + }; + } + return undefined; +} + +function getCodeExecutionResultStreamParts( + parts: z.infer['parts'], +): LanguageModelV2StreamPart[] { + const resultParts: LanguageModelV2StreamPart[] = []; + parts?.forEach(part => { + if ('codeExecutionResult' in part && part.codeExecutionResult != null && part.codeExecutionResult.output != null) { + // Ensure output might be empty but outcome is present + const outputText = part.codeExecutionResult.output; + // Only create a part if there's an outcome, even if output is empty. + if (part.codeExecutionResult.outcome.length > 0) { + const formattedText = `Execution Result (Outcome: ${part.codeExecutionResult.outcome}):\n${outputText}`; + resultParts.push({ type: 'text', text: formattedText }); + } + } + }); + return resultParts; } function getInlineDataParts(parts: z.infer['parts']) { @@ -459,6 +555,32 @@ function getInlineDataParts(parts: z.infer['parts']) { ); } +function getReasoningParts( + parts: z.infer['parts'], +): LanguageModelV2Content | undefined { + const reasoningContentParts: string[] = []; + parts?.forEach(part => { + if ( + 'text' in part && + (part as { thought?: boolean }).thought === true && + part.text != null && + part.text.length > 0 + ) { + reasoningContentParts.push(part.text) + } + }); + if (reasoningContentParts.length === 0) { + return undefined; + } + + // Join reasoning segments + return { + type: 'reasoning', + text: reasoningContentParts.join('') + } +} + + function extractSources({ groundingMetadata, generateId, @@ -483,6 +605,24 @@ function extractSources({ })); } +const executableCodePartSchema = z.object({ + executableCode: z + .object({ + language: z.string(), + code: z.string(), + }) + .nullish(), +}); + +const codeExecutionResultPartSchema = z.object({ + codeExecutionResult: z + .object({ + outcome: z.string(), + output: z.string(), + }) + .nullish(), +}); + const contentSchema = z.object({ role: z.string(), parts: z @@ -490,6 +630,7 @@ const contentSchema = z.object({ z.union([ z.object({ text: z.string(), + thought: z.boolean().nullish(), }), z.object({ functionCall: z.object({ @@ -503,6 +644,8 @@ const contentSchema = z.object({ data: z.string(), }), }), + executableCodePartSchema, + codeExecutionResultPartSchema, ]), ) .nullish(), diff --git a/packages/google/src/google-generative-ai-options.ts b/packages/google/src/google-generative-ai-options.ts index 3715b4fe4231..4a8cd8841834 100644 --- a/packages/google/src/google-generative-ai-options.ts +++ b/packages/google/src/google-generative-ai-options.ts @@ -51,6 +51,15 @@ export const googleGenerativeAIProviderOptions = z.object({ thinkingConfig: z .object({ thinkingBudget: z.number().optional(), + /** + * Optional. Set to true to include thinking process information in the response. + * This is primarily for use with Google Vertex AI, as behavior with other + * Google Generative AI endpoints might vary or not be fully supported. + * + * @see https://ai.google.dev/gemini-api/docs/thinking (for general concept) + * @see https://cloud.google.com/vertex-ai/generative-ai/docs/thinking (Vertex specific) + */ + includeThoughts: z.boolean().optional(), }) .optional(), @@ -130,6 +139,17 @@ Optional. Specifies the dynamic retrieval configuration. @see https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/ground-with-google-search#dynamic-retrieval */ dynamicRetrievalConfig: dynamicRetrievalConfig.optional(), + /** +Optional. When enabled, the model will make use of a code execution tool that +enables the model to generate and run Python code. + +@note Ensure the selected model supports Code Execution. +Multi-tool usage with the code execution tool is typically compatible with Flash experimental models. + +@see https://ai.google.dev/gemini-api/docs/code-execution (Google AI) +@see https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/code-execution-api (Vertex AI) + */ + useCodeExecution: z.boolean().optional(), }); export type GoogleGenerativeAIProviderOptions = z.infer< diff --git a/packages/google/src/google-prepare-tools.ts b/packages/google/src/google-prepare-tools.ts index d7434cb17ac4..4d20bb5261e5 100644 --- a/packages/google/src/google-prepare-tools.ts +++ b/packages/google/src/google-prepare-tools.ts @@ -15,36 +15,39 @@ export function prepareTools({ useSearchGrounding, dynamicRetrievalConfig, modelId, + useCodeExecution, }: { tools: LanguageModelV2CallOptions['tools']; toolChoice?: LanguageModelV2CallOptions['toolChoice']; useSearchGrounding: boolean; dynamicRetrievalConfig: DynamicRetrievalConfig | undefined; modelId: GoogleGenerativeAIModelId; + useCodeExecution: boolean; }): { tools: - | undefined - | { - functionDeclarations: Array<{ - name: string; - description: string | undefined; - parameters: unknown; - }>; - } - | { - googleSearchRetrieval: - | Record - | { dynamicRetrievalConfig: DynamicRetrievalConfig }; - } - | { googleSearch: Record }; + | undefined + | { + functionDeclarations: Array<{ + name: string; + description: string | undefined; + parameters: unknown; + }>; + } + | { + googleSearchRetrieval: + | Record + | { dynamicRetrievalConfig: DynamicRetrievalConfig }; + } + | { googleSearch: Record } + | { codeExecution: Record }; toolConfig: - | undefined - | { - functionCallingConfig: { - mode: 'AUTO' | 'NONE' | 'ANY'; - allowedFunctionNames?: string[]; - }; - }; + | undefined + | { + functionCallingConfig: { + mode: 'AUTO' | 'NONE' | 'ANY'; + allowedFunctionNames?: string[]; + }; + }; toolWarnings: LanguageModelV2CallWarning[]; } { // when the tools array is empty, change it to undefined to prevent errors: @@ -56,16 +59,48 @@ export function prepareTools({ const supportsDynamicRetrieval = modelId.includes('gemini-1.5-flash') && !modelId.includes('-8b'); + if ((useSearchGrounding || useCodeExecution) && tools) { + throw new UnsupportedFunctionalityError({ + functionality: + 'Provider-defined tools (useSearchGrounding or useCodeExecution) ' + + 'cannot be used in combination with user-defined tools. ' + + 'Please disable either the provider tools or your custom tools.', + }); + } + + // Ensure mutual exclusivity of provider-defined tools + if (useSearchGrounding && useCodeExecution) { + throw new UnsupportedFunctionalityError({ + functionality: + 'useSearchGrounding and useCodeExecution cannot be enabled simultaneously for this API version.', + }); + } + + if (useCodeExecution) { + // Add model compatibility check for code execution if necessary + // For example, if only specific models support it: + if (!isGemini2) { // Replace with actual model check for code execution + throw new UnsupportedFunctionalityError({ + functionality: `Code Execution is not supported for model ${modelId}. It requires a Gemini 2 or compatible model.`, + }); + } + return { + tools: { codeExecution: {} }, + toolConfig: undefined, + toolWarnings, + }; + } + if (useSearchGrounding) { return { tools: isGemini2 ? { googleSearch: {} } : { - googleSearchRetrieval: - !supportsDynamicRetrieval || !dynamicRetrievalConfig - ? {} - : { dynamicRetrievalConfig }, - }, + googleSearchRetrieval: + !supportsDynamicRetrieval || !dynamicRetrievalConfig + ? {} + : { dynamicRetrievalConfig }, + }, toolConfig: undefined, toolWarnings, }; From 2d15a6f201d437d63f68ea910eddf41b6ce020e7 Mon Sep 17 00:00:00 2001 From: Und3rf10w Date: Fri, 16 May 2025 19:35:28 -0400 Subject: [PATCH 05/10] tweak(providers/google): typo in codeExecutionResult part --- packages/google/src/google-generative-ai-language-model.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/google/src/google-generative-ai-language-model.ts b/packages/google/src/google-generative-ai-language-model.ts index 7733d8d8a3e3..01e759f9d01c 100644 --- a/packages/google/src/google-generative-ai-language-model.ts +++ b/packages/google/src/google-generative-ai-language-model.ts @@ -234,7 +234,7 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { } else if ('codeExecutionResult' in part && part.codeExecutionResult != null && part.codeExecutionResult.outcome.length > 0) { content.push({ type: 'text' as const, - text: `Code Execution Result (Outcome: ${part.codeExecutionResult.outcome}):\n ${part.codeExecutionResult.output}`, + text: `Execution Result (Outcome: ${part.codeExecutionResult.outcome}):\n ${part.codeExecutionResult.output}`, }) // function calls } else if ('functionCall' in part) { From 046c7d6b33e7850f4d18f365edc3a4c8e9fd8bc0 Mon Sep 17 00:00:00 2001 From: Und3rf10w Date: Fri, 16 May 2025 19:41:30 -0400 Subject: [PATCH 06/10] chore: Prettier-fixes --- .../generate-text/google-code-execution.ts | 81 +++++++------- .../google-vertex-code-execution.ts | 80 +++++++------- .../generate-text/google-vertex-reasoning.ts | 2 +- .../src/stream-text/google-code-execution.ts | 85 ++++++++------- .../google-vertex-code-execution.ts | 84 +++++++-------- .../stream-text/google-vertex-reasoning.ts | 2 +- .../google-generative-ai-language-model.ts | 102 +++++++++++------- packages/google/src/google-prepare-tools.ts | 57 +++++----- 8 files changed, 261 insertions(+), 232 deletions(-) diff --git a/examples/ai-core/src/generate-text/google-code-execution.ts b/examples/ai-core/src/generate-text/google-code-execution.ts index e4adea90afda..1575c1468258 100644 --- a/examples/ai-core/src/generate-text/google-code-execution.ts +++ b/examples/ai-core/src/generate-text/google-code-execution.ts @@ -1,54 +1,53 @@ - import { google } from '@ai-sdk/google'; import { generateText } from 'ai'; import 'dotenv/config'; async function main() { - const result = await generateText({ - model: google('gemini-2.5-flash-preview-04-17'), - providerOptions: { - google: { - useCodeExecution: true, - } - }, - maxOutputTokens: 2048, - prompt: - 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', - }); + const result = await generateText({ + model: google('gemini-2.5-flash-preview-04-17'), + providerOptions: { + google: { + useCodeExecution: true, + }, + }, + maxOutputTokens: 2048, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); - for (const part of result.content) { - switch (part.type) { - case 'file': { - if (part.type === 'file') { - process.stdout.write( - '\x1b[33m' + - part.type + - '\x1b[34m: ' + - part.file.mediaType + - '\x1b[0m' - ); - console.log(); - console.log(atob(part.file.base64)) - } - } - case 'text': { - if (part.type === 'text') { - process.stdout.write('\x1b[34m' + part.type + '\x1b[0m'); - console.log(); - console.log(part.text) - } - } + for (const part of result.content) { + switch (part.type) { + case 'file': { + if (part.type === 'file') { + process.stdout.write( + '\x1b[33m' + + part.type + + '\x1b[34m: ' + + part.file.mediaType + + '\x1b[0m', + ); + console.log(); + console.log(atob(part.file.base64)); + } + } + case 'text': { + if (part.type === 'text') { + process.stdout.write('\x1b[34m' + part.type + '\x1b[0m'); + console.log(); + console.log(part.text); } + } } + } - process.stdout.write('\n\n'); + process.stdout.write('\n\n'); - console.log(); - console.log('Warnings:', await result.warnings); + console.log(); + console.log('Warnings:', await result.warnings); - console.log(); - console.log('Token usage:', await result.usage); - console.log('Finish reason:', await result.finishReason); + console.log(); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); } -main().catch(console.error); \ No newline at end of file +main().catch(console.error); diff --git a/examples/ai-core/src/generate-text/google-vertex-code-execution.ts b/examples/ai-core/src/generate-text/google-vertex-code-execution.ts index 63e10a937126..8759517a2319 100644 --- a/examples/ai-core/src/generate-text/google-vertex-code-execution.ts +++ b/examples/ai-core/src/generate-text/google-vertex-code-execution.ts @@ -3,51 +3,51 @@ import { generateText } from 'ai'; import 'dotenv/config'; async function main() { - const result = await generateText({ - model: vertex('gemini-2.5-pro-preview-05-06'), - providerOptions: { - google: { - useCodeExecution: true, - } - }, - maxOutputTokens: 2048, - prompt: - 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', - }); + const result = await generateText({ + model: vertex('gemini-2.5-pro-preview-05-06'), + providerOptions: { + google: { + useCodeExecution: true, + }, + }, + maxOutputTokens: 2048, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); - for (const part of result.content) { - switch (part.type) { - case 'file': { - if (part.type === 'file') { - process.stdout.write( - '\x1b[33m' + - part.type + - '\x1b[34m: ' + - part.file.mediaType + - '\x1b[0m' - ); - console.log(); - console.log(atob(part.file.base64)) - } - } - case 'text': { - if (part.type === 'text') { - process.stdout.write('\x1b[34m' + part.type + '\x1b[0m'); - console.log(); - console.log(part.text) - } - } + for (const part of result.content) { + switch (part.type) { + case 'file': { + if (part.type === 'file') { + process.stdout.write( + '\x1b[33m' + + part.type + + '\x1b[34m: ' + + part.file.mediaType + + '\x1b[0m', + ); + console.log(); + console.log(atob(part.file.base64)); } + } + case 'text': { + if (part.type === 'text') { + process.stdout.write('\x1b[34m' + part.type + '\x1b[0m'); + console.log(); + console.log(part.text); + } + } } + } - process.stdout.write('\n\n'); + process.stdout.write('\n\n'); - console.log(); - console.log('Warnings:', await result.warnings); + console.log(); + console.log('Warnings:', await result.warnings); - console.log(); - console.log('Token usage:', await result.usage); - console.log('Finish reason:', await result.finishReason); + console.log(); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); } -main().catch(console.error); \ No newline at end of file +main().catch(console.error); diff --git a/examples/ai-core/src/generate-text/google-vertex-reasoning.ts b/examples/ai-core/src/generate-text/google-vertex-reasoning.ts index 3b6b568a0b2b..0d0c025fb74c 100644 --- a/examples/ai-core/src/generate-text/google-vertex-reasoning.ts +++ b/examples/ai-core/src/generate-text/google-vertex-reasoning.ts @@ -9,7 +9,7 @@ async function main() { google: { thinkingConfig: { thinkingBudget: 1024, - includeThoughts: true + includeThoughts: true, }, }, }, diff --git a/examples/ai-core/src/stream-text/google-code-execution.ts b/examples/ai-core/src/stream-text/google-code-execution.ts index b27a816fc260..ad7067542528 100644 --- a/examples/ai-core/src/stream-text/google-code-execution.ts +++ b/examples/ai-core/src/stream-text/google-code-execution.ts @@ -1,55 +1,54 @@ - import { google } from '@ai-sdk/google'; import { streamText } from 'ai'; import 'dotenv/config'; async function main() { - const result = streamText({ - model: google('gemini-2.5-flash-preview-04-17'), - maxOutputTokens: 10000, - providerOptions: { - google: { - useCodeExecution: true - } - }, - prompt: - 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', - }); + const result = streamText({ + model: google('gemini-2.5-flash-preview-04-17'), + maxOutputTokens: 10000, + providerOptions: { + google: { + useCodeExecution: true, + }, + }, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); - let fullResponse = ''; + let fullResponse = ''; - for await (const delta of result.fullStream) { - switch (delta.type) { - case 'file': { - if (delta.type === 'file') { - process.stdout.write( - '\x1b[33m' + - delta.type + - '\x1b[34m: ' + - delta.file.mediaType + - '\x1b[0m' - ); - console.log(); - console.log(atob(delta.file.base64 as string)); - } - } - case 'text': { - if (delta.type === 'text') { - process.stdout.write('\x1b[34m' + delta.type + '\x1b[0m'); - console.log(); - console.log(delta.text); - fullResponse += delta.text; - } - break; - } + for await (const delta of result.fullStream) { + switch (delta.type) { + case 'file': { + if (delta.type === 'file') { + process.stdout.write( + '\x1b[33m' + + delta.type + + '\x1b[34m: ' + + delta.file.mediaType + + '\x1b[0m', + ); + console.log(); + console.log(atob(delta.file.base64 as string)); + } + } + case 'text': { + if (delta.type === 'text') { + process.stdout.write('\x1b[34m' + delta.type + '\x1b[0m'); + console.log(); + console.log(delta.text); + fullResponse += delta.text; } + break; + } } - console.log(); - console.log('Warnings:', await result.warnings); + } + console.log(); + console.log('Warnings:', await result.warnings); - console.log(); - console.log('Token usage:', await result.usage); - console.log('Finish reason:', await result.finishReason); + console.log(); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); } -main().catch(console.log); \ No newline at end of file +main().catch(console.log); diff --git a/examples/ai-core/src/stream-text/google-vertex-code-execution.ts b/examples/ai-core/src/stream-text/google-vertex-code-execution.ts index f8b1bfbfd211..68b8b5222ab6 100644 --- a/examples/ai-core/src/stream-text/google-vertex-code-execution.ts +++ b/examples/ai-core/src/stream-text/google-vertex-code-execution.ts @@ -3,52 +3,52 @@ import { streamText } from 'ai'; import 'dotenv/config'; async function main() { - const result = streamText({ - model: vertex('gemini-2.5-flash-preview-04-17'), - providerOptions: { - google: { - useCodeExecution: true - } - }, - maxOutputTokens: 10000, - prompt: - 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', - }); + const result = streamText({ + model: vertex('gemini-2.5-flash-preview-04-17'), + providerOptions: { + google: { + useCodeExecution: true, + }, + }, + maxOutputTokens: 10000, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); - let fullResponse = ''; + let fullResponse = ''; - for await (const delta of result.fullStream) { - switch (delta.type) { - case 'file': { - if (delta.type === 'file') { - process.stdout.write( - '\x1b[33m' + - delta.type + - '\x1b[34m: ' + - delta.file.mediaType + - '\x1b[0m' - ); - console.log(); - console.log(atob(delta.file.base64 as string)); - } - } - case 'text': { - if (delta.type === 'text') { - process.stdout.write('\x1b[34m' + delta.type + '\x1b[0m'); - console.log(); - console.log(delta.text); - fullResponse += delta.text; - } - break; - } + for await (const delta of result.fullStream) { + switch (delta.type) { + case 'file': { + if (delta.type === 'file') { + process.stdout.write( + '\x1b[33m' + + delta.type + + '\x1b[34m: ' + + delta.file.mediaType + + '\x1b[0m', + ); + console.log(); + console.log(atob(delta.file.base64 as string)); } + } + case 'text': { + if (delta.type === 'text') { + process.stdout.write('\x1b[34m' + delta.type + '\x1b[0m'); + console.log(); + console.log(delta.text); + fullResponse += delta.text; + } + break; + } } - console.log(); - console.log('Warnings:', await result.warnings); + } + console.log(); + console.log('Warnings:', await result.warnings); - console.log(); - console.log('Token usage:', await result.usage); - console.log('Finish reason:', await result.finishReason); + console.log(); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); } -main().catch(console.log); \ No newline at end of file +main().catch(console.log); diff --git a/examples/ai-core/src/stream-text/google-vertex-reasoning.ts b/examples/ai-core/src/stream-text/google-vertex-reasoning.ts index 61455643d641..aa9e445c1681 100644 --- a/examples/ai-core/src/stream-text/google-vertex-reasoning.ts +++ b/examples/ai-core/src/stream-text/google-vertex-reasoning.ts @@ -10,7 +10,7 @@ async function main() { google: { thinkingConfig: { thinkingBudget: 1024, - includeThoughts: true + includeThoughts: true, }, }, }, diff --git a/packages/google/src/google-generative-ai-language-model.ts b/packages/google/src/google-generative-ai-language-model.ts index 01e759f9d01c..df62e970cee9 100644 --- a/packages/google/src/google-generative-ai-language-model.ts +++ b/packages/google/src/google-generative-ai-language-model.ts @@ -104,7 +104,6 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { }); } - const { contents, systemInstruction } = convertToGoogleGenerativeAIMessages(prompt); @@ -141,11 +140,11 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { responseFormat?.type === 'json' ? 'application/json' : undefined, responseSchema: responseFormat?.type === 'json' && - responseFormat.schema != null && - // Google GenAI does not support all OpenAPI Schema features, - // so this is needed as an escape hatch: - // TODO convert into provider option - (googleOptions?.structuredOutputs ?? true) + responseFormat.schema != null && + // Google GenAI does not support all OpenAPI Schema features, + // so this is needed as an escape hatch: + // TODO convert into provider option + (googleOptions?.structuredOutputs ?? true) ? convertJSONSchemaToOpenAPISchema(responseFormat.schema) : undefined, ...(googleOptions?.audioTimestamp && { @@ -200,8 +199,8 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { // map ordered parts to content: const parts = candidate.content == null || - typeof candidate.content !== 'object' || - !('parts' in candidate.content) + typeof candidate.content !== 'object' || + !('parts' in candidate.content) ? [] : (candidate.content.parts ?? []); @@ -220,7 +219,11 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { ) { content.push({ type: 'reasoning', text: part.text }); // code exectuion: Executable code - } else if ('executableCode' in part && part.executableCode != null && part.executableCode.code.length > 0) { + } else if ( + 'executableCode' in part && + part.executableCode != null && + part.executableCode.code.length > 0 + ) { /** * NOTE: The vertex api just returns a string, but the ai-sdk expects either a base64 string or uint8Arry. * So we just convert it to base64 @@ -228,14 +231,20 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { content.push({ type: 'file', mediaType: 'text/x-python', - data: Buffer.from(part.executableCode.code, 'utf-8').toString('base64') - }) + data: Buffer.from(part.executableCode.code, 'utf-8').toString( + 'base64', + ), + }); // code execution: Execution result - } else if ('codeExecutionResult' in part && part.codeExecutionResult != null && part.codeExecutionResult.outcome.length > 0) { + } else if ( + 'codeExecutionResult' in part && + part.codeExecutionResult != null && + part.codeExecutionResult.outcome.length > 0 + ) { content.push({ type: 'text' as const, text: `Execution Result (Outcome: ${part.codeExecutionResult.outcome}):\n ${part.codeExecutionResult.output}`, - }) + }); // function calls } else if ('functionCall' in part) { content.push({ @@ -373,17 +382,25 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { // Process tool call's parts before determining finishReason to ensure hasToolCalls is properly set if (content != null) { const reasoningContent = getReasoningParts(content.parts); - if (reasoningContent?.type === 'reasoning' && reasoningContent.text.length > 0) { - controller.enqueue({ type: 'reasoning', text: reasoningContent.text }); + if ( + reasoningContent?.type === 'reasoning' && + reasoningContent.text.length > 0 + ) { + controller.enqueue({ + type: 'reasoning', + text: reasoningContent.text, + }); } // Process code execution parts prior to text parts - const executableCodeFilePart = getExecutableCodeFilePartFromStreamParts(content.parts); + const executableCodeFilePart = + getExecutableCodeFilePartFromStreamParts(content.parts); if (executableCodeFilePart != null) { controller.enqueue(executableCodeFilePart); } - const codeExecutionResultTextParts = getCodeExecutionResultStreamParts(content.parts); + const codeExecutionResultTextParts = + getCodeExecutionResultStreamParts(content.parts); for (const textPart of codeExecutionResultTextParts) { controller.enqueue(textPart); } @@ -491,33 +508,43 @@ function getToolCallsFromParts({ return functionCallParts == null || functionCallParts.length === 0 ? undefined : functionCallParts.map(part => ({ - type: 'tool-call' as const, - toolCallType: 'function' as const, - toolCallId: generateId(), - toolName: part.functionCall.name, - args: JSON.stringify(part.functionCall.args), - })); + type: 'tool-call' as const, + toolCallType: 'function' as const, + toolCallId: generateId(), + toolName: part.functionCall.name, + args: JSON.stringify(part.functionCall.args), + })); } function getTextFromParts(parts: z.infer['parts']) { // Only include plain text parts (not thoughts, not executable code, not code execution results) - const textParts = parts?.filter(part => 'text' in part && part.text != null && !(part as { thought?: boolean }).thought) as Array< - GoogleGenerativeAIContentPart & { text: string } - >; + const textParts = parts?.filter( + part => + 'text' in part && + part.text != null && + !(part as { thought?: boolean }).thought, + ) as Array; return textParts == null || textParts.length === 0 ? undefined : { - type: 'text' as const, - text: textParts.map(part => part.text).join(''), - }; + type: 'text' as const, + text: textParts.map(part => part.text).join(''), + }; } function getExecutableCodeFilePartFromStreamParts( parts: z.infer['parts'], ): LanguageModelV2StreamPart | undefined { - const part = parts?.find(p => 'executableCode' in p && p.executableCode != null); - if (part && 'executableCode' in part && part.executableCode && part.executableCode.code.length > 0) { + const part = parts?.find( + p => 'executableCode' in p && p.executableCode != null, + ); + if ( + part && + 'executableCode' in part && + part.executableCode && + part.executableCode.code.length > 0 + ) { return { type: 'file', mediaType: 'text/x-python', @@ -532,7 +559,11 @@ function getCodeExecutionResultStreamParts( ): LanguageModelV2StreamPart[] { const resultParts: LanguageModelV2StreamPart[] = []; parts?.forEach(part => { - if ('codeExecutionResult' in part && part.codeExecutionResult != null && part.codeExecutionResult.output != null) { + if ( + 'codeExecutionResult' in part && + part.codeExecutionResult != null && + part.codeExecutionResult.output != null + ) { // Ensure output might be empty but outcome is present const outputText = part.codeExecutionResult.output; // Only create a part if there's an outcome, even if output is empty. @@ -566,7 +597,7 @@ function getReasoningParts( part.text != null && part.text.length > 0 ) { - reasoningContentParts.push(part.text) + reasoningContentParts.push(part.text); } }); if (reasoningContentParts.length === 0) { @@ -576,11 +607,10 @@ function getReasoningParts( // Join reasoning segments return { type: 'reasoning', - text: reasoningContentParts.join('') - } + text: reasoningContentParts.join(''), + }; } - function extractSources({ groundingMetadata, generateId, diff --git a/packages/google/src/google-prepare-tools.ts b/packages/google/src/google-prepare-tools.ts index 4d20bb5261e5..1a915ffcfb13 100644 --- a/packages/google/src/google-prepare-tools.ts +++ b/packages/google/src/google-prepare-tools.ts @@ -25,29 +25,29 @@ export function prepareTools({ useCodeExecution: boolean; }): { tools: - | undefined - | { - functionDeclarations: Array<{ - name: string; - description: string | undefined; - parameters: unknown; - }>; - } - | { - googleSearchRetrieval: - | Record - | { dynamicRetrievalConfig: DynamicRetrievalConfig }; - } - | { googleSearch: Record } - | { codeExecution: Record }; + | undefined + | { + functionDeclarations: Array<{ + name: string; + description: string | undefined; + parameters: unknown; + }>; + } + | { + googleSearchRetrieval: + | Record + | { dynamicRetrievalConfig: DynamicRetrievalConfig }; + } + | { googleSearch: Record } + | { codeExecution: Record }; toolConfig: - | undefined - | { - functionCallingConfig: { - mode: 'AUTO' | 'NONE' | 'ANY'; - allowedFunctionNames?: string[]; - }; - }; + | undefined + | { + functionCallingConfig: { + mode: 'AUTO' | 'NONE' | 'ANY'; + allowedFunctionNames?: string[]; + }; + }; toolWarnings: LanguageModelV2CallWarning[]; } { // when the tools array is empty, change it to undefined to prevent errors: @@ -79,7 +79,8 @@ export function prepareTools({ if (useCodeExecution) { // Add model compatibility check for code execution if necessary // For example, if only specific models support it: - if (!isGemini2) { // Replace with actual model check for code execution + if (!isGemini2) { + // Replace with actual model check for code execution throw new UnsupportedFunctionalityError({ functionality: `Code Execution is not supported for model ${modelId}. It requires a Gemini 2 or compatible model.`, }); @@ -96,11 +97,11 @@ export function prepareTools({ tools: isGemini2 ? { googleSearch: {} } : { - googleSearchRetrieval: - !supportsDynamicRetrieval || !dynamicRetrievalConfig - ? {} - : { dynamicRetrievalConfig }, - }, + googleSearchRetrieval: + !supportsDynamicRetrieval || !dynamicRetrievalConfig + ? {} + : { dynamicRetrievalConfig }, + }, toolConfig: undefined, toolWarnings, }; From 29ce45ab04a6775edc03622549e2e199747116e1 Mon Sep 17 00:00:00 2001 From: Und3rf10w Date: Fri, 16 May 2025 19:43:09 -0400 Subject: [PATCH 07/10] chore(providers/google): Added changeset --- .changeset/thin-eagles-serve.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/thin-eagles-serve.md diff --git a/.changeset/thin-eagles-serve.md b/.changeset/thin-eagles-serve.md new file mode 100644 index 000000000000..510725348c37 --- /dev/null +++ b/.changeset/thin-eagles-serve.md @@ -0,0 +1,7 @@ +--- +'@ai-sdk/google-vertex': patch +'@example/ai-core': patch +'@ai-sdk/google': patch +--- + +Added Reasoning and Code Execution support to google providers From 73e10c2617fa48742b6f07d7441d6bbe54c3ae04 Mon Sep 17 00:00:00 2001 From: Und3rf10w Date: Fri, 16 May 2025 19:54:15 -0400 Subject: [PATCH 08/10] chore(docs/providers): Update google providers with codeExecution info --- .../15-google-generative-ai.mdx | 192 ++++++++++++++++++ .../01-ai-sdk-providers/16-google-vertex.mdx | 188 +++++++++++++++++ 2 files changed, 380 insertions(+) diff --git a/content/providers/01-ai-sdk-providers/15-google-generative-ai.mdx b/content/providers/01-ai-sdk-providers/15-google-generative-ai.mdx index 92078eb6f2e1..18677de7c9c2 100644 --- a/content/providers/01-ai-sdk-providers/15-google-generative-ai.mdx +++ b/content/providers/01-ai-sdk-providers/15-google-generative-ai.mdx @@ -143,6 +143,14 @@ The following optional provider options are available for Google Generative AI m - `BLOCK_ONLY_HIGH` - `BLOCK_NONE` +- **useSearchGrounding** _boolean_ + + Optional. When enabled, the model will [use Google search to ground the response](https://ai.google.dev/gemini-api/docs/grounding). + +- **useCodeExecution** _boolean_ + + Optional. When enabled, the model will make use of a code execution tool that [enables the model to generate and run Python code](https://ai.google.dev/gemini-api/docs/code-execution). + - **responseModalities** _string[]_ The modalities to use for the response. The following modalities are supported: `TEXT`, `IMAGE`. When not defined or empty, the model defaults to returning only text. @@ -396,6 +404,190 @@ const { sources } = await generateText({ }); ``` +### Code Execution + +With [Code Execution](https://ai.google.dev/gemini-api/docs/code-execution), certain models can generate and execute Python code to perform calculations, solve problems, or provide more accurate information. + +To enable this feature, set `useCodeExecution: true` in the `providerOptions` for the Google provider: + +```ts highlight="6-10" +import { google } from '@ai-sdk/google'; +import { generateText } from 'ai'; + +async function main() { + const result = await generateText({ + model: google('gemini-2.5-flash-preview-04-17'), + providerOptions: { + google: { + useCodeExecution: true, + }, + }, + prompt: + 'Calculate the 20th Fibonacci number. Then find the nearest palindrome to it.', + }); + + // Process result.content which may include file and text parts + // (see example below) + console.log('Final aggregated text:', result.text); +} + +main(); +``` + +When Code Execution is enabled, the model's response will surface the generated code and its execution results as distinct parts within the output: + +- **Generated Python Code**: This is represented as a `file` content part (or stream part). + - `type`: `'file'` + - `mediaType`: `'text/x-python'` + - `data`: A base64-encoded string of the Python code that the model generated and executed. +- **Code Execution Result**: This is represented as a `text` content part (or stream part). + - `type`: `'text'` + - `text`: A formatted string detailing the execution `outcome` (e.g., "OUTCOME_OK") and the `output` from the code. The format is typically: `Execution Result (Outcome: ):\n`. + +#### `generateText` with Code Execution + +When using `generateText`, the `result.content` array will contain these `file` (for executable code) and `text` (for execution results) parts interspersed with other text parts generated by the model. + +Here's how you can process these parts: + +```ts +import { google } from '@ai-sdk/google'; +import { generateText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = await generateText({ + model: google('gemini-2.5-flash-preview-04-17'), + providerOptions: { + google: { + useCodeExecution: true, + }, + }, + maxOutputTokens: 2048, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); + + console.log('Processing content parts:'); + for (const part of result.content) { + switch (part.type) { + case 'file': { + // This is the executableCode part + process.stdout.write( + '\x1b[33m' + // Yellow color for "file" + part.type + + '\x1b[34m: ' + // Blue color for mediaType + part.mediaType + // Should be 'text/x-python' + '\x1b[0m', // Reset color + ); + console.log(); // Newline + // Data is base64 encoded Python code + console.log('Code:\n', atob(part.data as string)); + break; + } + case 'text': { + // This can be a regular text part or a codeExecutionResult + process.stdout.write( + '\x1b[34m' + part.type + '\x1b[0m', // Blue color for "text" + ); + console.log(); // Newline + console.log(part.text); // Contains model's text or formatted execution result + break; + } + } + } + + process.stdout.write('\n\n--- Full Response Details ---\n'); + console.log('Aggregated Text:', result.text); + console.log('Warnings:', result.warnings); + console.log('Token usage:', result.usage); + console.log('Finish reason:', result.finishReason); +} + +main().catch(console.error); +``` + +#### Streaming Code Execution Details (`streamText`) + +When using `streamText` with `useCodeExecution: true`, the generated Python code and its execution results are streamed as distinct part types: + +- **Generated Python Code**: Arrives as a stream part where `delta.type === 'file'`. + - `delta.mediaType` will be `'text/x-python'`. + - `delta.data` will be the base64-encoded Python code string. +- **Code Execution Result**: Arrives as a stream part where `delta.type === 'text'`. + - `delta.text` will contain the formatted string with the execution outcome and output (e.g., `Execution Result (Outcome: OUTCOME_OK):\nOutput...`). + +Here's an example of how you might process the stream: + +```ts +import { google } from '@ai-sdk/google'; +import { streamText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = streamText({ + model: google('gemini-2.5-flash-preview-04-17'), + providerOptions: { + google: { + useCodeExecution: true, + }, + }, + maxOutputTokens: 10000, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); + + let fullResponse = ''; + console.log('Streaming content parts:'); + + for await (const delta of result.fullStream) { + switch (delta.type) { + case 'file': { + // This is the executableCode part + process.stdout.write( + '\x1b[33m' + // Yellow color for "file" + delta.type + + '\x1b[34m: ' + // Blue color for mediaType + delta.mediaType + // Should be 'text/x-python' + '\x1b[0m', // Reset color + ); + console.log(); // Newline + // Data is base64 encoded Python code + console.log('Code:\n', atob(delta.data as string)); + break; + } + case 'text': { + // This can be a regular text part or a codeExecutionResult + process.stdout.write( + '\x1b[34m' + delta.type + '\x1b[0m', // Blue color for "text" + ); + console.log(); // Newline + console.log(delta.text); // Contains model's text or formatted execution result + fullResponse += delta.text; + break; + } + // Other stream part types like 'reasoning', 'tool-call-delta', 'tool-call', + // 'stream-start', 'finish', 'error' can be handled here if needed. + } + } + + process.stdout.write('\n\n--- Full Response Details ---\n'); + console.log('Aggregated Text from Stream:', fullResponse); + console.log('Warnings:', await result.warnings); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); +} + +main().catch(console.error); +``` + + + Code Execution capabilities and specific model support are subject to Google's + offerings. Always refer to the [official Google AI + documentation](https://ai.google.dev/gemini-api/docs/code-execution) for the + most current information on compatible models and features. + + ### Image Outputs The model `gemini-2.0-flash-exp` supports image generation. Images are exposed as files in the response. diff --git a/content/providers/01-ai-sdk-providers/16-google-vertex.mdx b/content/providers/01-ai-sdk-providers/16-google-vertex.mdx index 4fdcad54ea4c..ec330f05f40d 100644 --- a/content/providers/01-ai-sdk-providers/16-google-vertex.mdx +++ b/content/providers/01-ai-sdk-providers/16-google-vertex.mdx @@ -304,6 +304,10 @@ The following optional provider options are available for Google Vertex models: Optional. When enabled, the model will [use Google search to ground the response](https://cloud.google.com/vertex-ai/generative-ai/docs/grounding/overview). +- **useCodeExecution** _boolean_ + + Optional. When enabled, the model will make use of a code execution tool that enables the model to [generate and run Python code](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/code-execution). + - **audioTimestamp** _boolean_ Optional. Enables timestamp understanding for audio files. Defaults to false. @@ -446,6 +450,190 @@ Example response excerpt: threshold](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/ground-gemini#dynamic-retrieval). +#### Code Execution + +With [Code Execution](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/code-execution), certain Gemini models on Vertex AI can generate and execute Python code. This allows the model to perform calculations, data manipulation, and other programmatic tasks to enhance its responses. + +To enable this feature, set `useCodeExecution: true` in the `providerOptions` for the Google provider when making a call: + +```ts highlight="6-10" +import { vertex } from '@ai-sdk/google-vertex'; +import { generateText } from 'ai'; + +async function main() { + const result = await generateText({ + model: vertex('gemini-2.5-pro-preview-05-06'), + providerOptions: { + google: { + // Note: providerOptions are nested under 'google' key for Vertex + useCodeExecution: true, + }, + }, + prompt: 'What is the result of 7 factorial divided by 3 factorial?', + }); + + // Process result.content which may include file and text parts + // (see example below) + console.log('Final aggregated text:', result.text); +} + +main(); +``` + +When Code Execution is active, the model's response will surface the generated code and its execution results as distinct parts within the output: + +- **Generated Python Code**: This is represented as a `file` content part (or stream part). + - `type`: `'file'` + - `mediaType`: `'text/x-python'` + - `data`: A base64-encoded string of the Python code that the model generated and executed. +- **Code Execution Result**: This is represented as a `text` content part (or stream part). + - `type`: `'text'` + - `text`: A formatted string detailing the execution `outcome` (e.g., "OUTCOME_OK") and the `output` from the code. The format is typically: `Execution Result (Outcome: ):\n`. + +##### `generateText` with Code Execution + +When using `generateText`, the `result.content` array will contain these `file` (for executable code) and `text` (for execution results) parts, potentially interspersed with other text parts generated by the model. + +Here's how you can process these parts: + +```ts +import { vertex } from '@ai-sdk/google-vertex'; +import { generateText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = await generateText({ + model: vertex('gemini-2.5-pro-preview-05-06'), + providerOptions: { + google: { + useCodeExecution: true, + }, + }, + maxOutputTokens: 2048, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); + + console.log('Processing content parts:'); + for (const part of result.content) { + switch (part.type) { + case 'file': { + // This is the executableCode part + process.stdout.write( + '\x1b[33m' + // Yellow color for "file" + part.type + + '\x1b[34m: ' + // Blue color for mediaType + part.mediaType + // Should be 'text/x-python' + '\x1b[0m', // Reset color + ); + console.log(); // Newline + // Data is base64 encoded Python code + console.log('Code:\n', atob(part.data as string)); + break; + } + case 'text': { + // This can be a regular text part or a codeExecutionResult + process.stdout.write( + '\x1b[34m' + part.type + '\x1b[0m', // Blue color for "text" + ); + console.log(); // Newline + console.log(part.text); // Contains model's text or formatted execution result + break; + } + } + } + + process.stdout.write('\n\n--- Full Response Details ---\n'); + console.log('Aggregated Text:', result.text); + console.log('Warnings:', result.warnings); + console.log('Token usage:', result.usage); + console.log('Finish reason:', result.finishReason); +} + +main().catch(console.error); +``` + +##### Streaming Code Execution Details (`streamText`) + +When using `streamText` with a Vertex model that has `useCodeExecution: true` enabled in `providerOptions`, the generated Python code and its execution results are streamed as distinct part types: + +- **Generated Python Code**: Arrives as a stream part where `delta.type === 'file'`. + - `delta.mediaType` will be `'text/x-python'`. + - `delta.data` will be the base64-encoded Python code string. +- **Code Execution Result**: Arrives as a stream part where `delta.type === 'text'`. + - `delta.text` will contain the formatted string with the execution outcome and output (e.g., `Execution Result (Outcome: OUTCOME_OK):\nOutput...`). + +Here's an example of how you might process the stream: + +```ts +import { vertex } from '@ai-sdk/google-vertex'; +import { streamText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = streamText({ + model: vertex('gemini-2.5-pro-preview-05-06'), + providerOptions: { + google: { + useCodeExecution: true, + }, + }, + maxOutputTokens: 10000, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it.', + }); + + let fullResponse = ''; + console.log('Streaming content parts:'); + + for await (const delta of result.fullStream) { + switch (delta.type) { + case 'file': { + // This is the executableCode part + process.stdout.write( + '\x1b[33m' + // Yellow color for "file" + delta.type + + '\x1b[34m: ' + // Blue color for mediaType + delta.mediaType + // Should be 'text/x-python' + '\x1b[0m', // Reset color + ); + console.log(); // Newline + // Data is base64 encoded Python code + console.log('Code:\n', atob(delta.data as string)); + break; + } + case 'text': { + // This can be a regular text part or a codeExecutionResult + process.stdout.write( + '\x1b[34m' + delta.type + '\x1b[0m', // Blue color for "text" + ); + console.log(); // Newline + console.log(delta.text); // Contains model's text or formatted execution result + fullResponse += delta.text; + break; + } + // Other stream part types like 'reasoning', 'tool-call-delta', 'tool-call', + // 'stream-start', 'finish', 'error' can be handled here if needed. + } + } + + process.stdout.write('\n\n--- Full Response Details ---\n'); + console.log('Aggregated Text from Stream:', fullResponse); + console.log('Warnings:', await result.warnings); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); +} + +main().catch(console.error); +``` + + + Code Execution capabilities and specific model support on Vertex AI are + subject to Google Cloud's offerings. Always refer to the [official Vertex AI + documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/code-execution) + for the most current information on compatible models and features. + + ### Sources When you use [Search Grounding](#search-grounding), the model will include sources in the response. From eb4bfc99953344a3bf30e7ac325737a5d38f13f2 Mon Sep 17 00:00:00 2001 From: Und3rf10w Date: Sat, 17 May 2025 15:10:25 -0400 Subject: [PATCH 09/10] feat(providers/google): Allow setting useSearchGrounding and useCodeExecution simultaneously with the google.generative-ai provider --- ...-reasoning-code-execution-and-grounding.ts | 136 ++++++++++++++++++ .../google-generative-ai-language-model.ts | 4 +- packages/google/src/google-prepare-tools.ts | 27 +++- 3 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 examples/ai-core/src/stream-text/google-reasoning-code-execution-and-grounding.ts diff --git a/examples/ai-core/src/stream-text/google-reasoning-code-execution-and-grounding.ts b/examples/ai-core/src/stream-text/google-reasoning-code-execution-and-grounding.ts new file mode 100644 index 000000000000..a9d11fd4fb75 --- /dev/null +++ b/examples/ai-core/src/stream-text/google-reasoning-code-execution-and-grounding.ts @@ -0,0 +1,136 @@ +import { google, GoogleGenerativeAIProviderMetadata } from '@ai-sdk/google'; +import { streamText } from 'ai'; +import 'dotenv/config'; + +async function main() { + const result = streamText({ + model: google('gemini-2.5-flash-preview-04-17'), + maxOutputTokens: 10000, + providerOptions: { + google: { + // Only GoogleGenerativeAI Provider supports both grounding and code execution + useCodeExecution: true, + useSearchGrounding: true, + // Flash Preview supports thinking + thinkingConfig: { + thinkingBudget: 2048, + }, + }, + }, + onError(error) { + console.error(error); + }, + prompt: + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it. Also, provide the current XMR to USD rate, provide sources.', + }); + + for await (const delta of result.fullStream) { + switch (delta.type) { + case 'file': { + if (delta.type === 'file') { + process.stdout.write( + '\x1b[33m' + + delta.type + + '\x1b[34m: ' + + delta.file.mediaType + + '\x1b[0m', + ); + console.log(); + console.log(atob(delta.file.base64 as string)); + } + } + case 'text': { + if (delta.type === 'text') { + process.stdout.write('\x1b[34m' + delta.type + '\x1b[0m'); + console.log(); + console.log(delta.text); + } + } + case 'source': { + if (delta.type === 'source' && delta.sourceType === 'url') { + process.stdout.write('\x1b[32m' + delta.type + '\x1b[0m'); + console.log(); + console.log('ID:', delta.id); + console.log('Title:', delta.title); + console.log('URL:', delta.url); + console.log(); + } + } + case 'reasoning': { + if (delta.type === 'reasoning') { + console.log('\x1b[34m' + delta.type); + console.log(); + console.log(delta.text); + console.log('\x1b[0m'); + } + } + case 'tool-call': { + if (delta.type === 'tool-call') { + console.log('\x1b[33m' + delta.type); + console.log( + 'TOOL CALL: ', + delta.toolName, + '(', + delta.toolCallId, + ')', + ); + console.log('Args: ', delta.args); + console.log('\x1b[0m'); + } + } + case 'tool-result': { + if (delta.type === 'tool-result') { + console.log('\x1b[37m' + delta.type); + console.log(); + console.log( + 'TOOL RESULT: ', + delta.toolName, + '(', + delta.toolCallId, + ')', + ); + console.log(delta.result); + console.log('\x1b[0m'); + } + } + case 'error': { + console.log('\x1b[31m' + delta.type + '\x1b[0m'); + console.log(); + if (delta.type === 'error' && delta.error != null) { + console.log(delta.error); + } + } + } + } + + // Show sources + const metadata = (await result.providerMetadata) as + | GoogleGenerativeAIProviderMetadata + | undefined; + if (metadata != null) { + console.log('\x1b[31m' + 'sources' + '\x1b[0m'); + if ( + metadata?.groundingMetadata?.webSearchQueries && + metadata?.groundingMetadata?.webSearchQueries?.length > 0 + ) { + console.log( + 'Web Queries: ', + metadata?.groundingMetadata?.webSearchQueries, + ); + } + if (metadata.groundingMetadata?.searchEntryPoint != null) { + console.log( + 'Search Entry Point: ', + JSON.stringify(metadata.groundingMetadata?.searchEntryPoint, null, 2), + ); + } + } + console.log(); + console.log('Warnings:', await result.warnings); + + console.log(); + console.log('Token usage:', await result.usage); + console.log('Finish reason:', await result.finishReason); +} + +main().catch(console.error); diff --git a/packages/google/src/google-generative-ai-language-model.ts b/packages/google/src/google-generative-ai-language-model.ts index df62e970cee9..037e675e7479 100644 --- a/packages/google/src/google-generative-ai-language-model.ts +++ b/packages/google/src/google-generative-ai-language-model.ts @@ -118,6 +118,7 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { dynamicRetrievalConfig: googleOptions?.dynamicRetrievalConfig, modelId: this.modelId, useCodeExecution: googleOptions?.useCodeExecution ?? false, + provider: this.provider, }); warnings.push(...toolWarnings); @@ -238,8 +239,7 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { // code execution: Execution result } else if ( 'codeExecutionResult' in part && - part.codeExecutionResult != null && - part.codeExecutionResult.outcome.length > 0 + part.codeExecutionResult != null ) { content.push({ type: 'text' as const, diff --git a/packages/google/src/google-prepare-tools.ts b/packages/google/src/google-prepare-tools.ts index 1a915ffcfb13..c907f79cc9f0 100644 --- a/packages/google/src/google-prepare-tools.ts +++ b/packages/google/src/google-prepare-tools.ts @@ -16,6 +16,7 @@ export function prepareTools({ dynamicRetrievalConfig, modelId, useCodeExecution, + provider, }: { tools: LanguageModelV2CallOptions['tools']; toolChoice?: LanguageModelV2CallOptions['toolChoice']; @@ -23,6 +24,7 @@ export function prepareTools({ dynamicRetrievalConfig: DynamicRetrievalConfig | undefined; modelId: GoogleGenerativeAIModelId; useCodeExecution: boolean; + provider: string; }): { tools: | undefined @@ -38,6 +40,10 @@ export function prepareTools({ | Record | { dynamicRetrievalConfig: DynamicRetrievalConfig }; } + | { + googleSearch: Record; + codeExecution: Record; + } | { googleSearch: Record } | { codeExecution: Record }; toolConfig: @@ -70,10 +76,23 @@ export function prepareTools({ // Ensure mutual exclusivity of provider-defined tools if (useSearchGrounding && useCodeExecution) { - throw new UnsupportedFunctionalityError({ - functionality: - 'useSearchGrounding and useCodeExecution cannot be enabled simultaneously for this API version.', - }); + if (provider !== 'google.generative-ai') { + throw new UnsupportedFunctionalityError({ + functionality: + 'useSearchGrounding and useCodeExecution only be enabled simultaneously with the Google Generative AI provider.', + }); + } + if (!isGemini2) { + throw new UnsupportedFunctionalityError({ + functionality: + 'useSearchGrounding cannot be used with useCodeExecution in Gemini <2 models.', + }); + } + return { + tools: { codeExecution: {}, googleSearch: {} }, + toolConfig: undefined, + toolWarnings, + }; } if (useCodeExecution) { From cbcd689e16f33dea487046c83c787637b1a71be3 Mon Sep 17 00:00:00 2001 From: Und3rf10w Date: Sat, 17 May 2025 15:43:26 -0400 Subject: [PATCH 10/10] fix(providers/google): Consistent part handlign for codeExecutionResult in generate and stream fix(examples/ai-core): Fix stream-text google codeExecution with searchGrounding example to provide web sources --- ...-reasoning-code-execution-and-grounding.ts | 69 ++++++++++--------- .../google-generative-ai-language-model.ts | 4 +- 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/examples/ai-core/src/stream-text/google-reasoning-code-execution-and-grounding.ts b/examples/ai-core/src/stream-text/google-reasoning-code-execution-and-grounding.ts index a9d11fd4fb75..4cb18e68a4d0 100644 --- a/examples/ai-core/src/stream-text/google-reasoning-code-execution-and-grounding.ts +++ b/examples/ai-core/src/stream-text/google-reasoning-code-execution-and-grounding.ts @@ -1,4 +1,5 @@ import { google, GoogleGenerativeAIProviderMetadata } from '@ai-sdk/google'; +import { GoogleGenerativeAIProviderOptions } from '@ai-sdk/google'; import { streamText } from 'ai'; import 'dotenv/config'; @@ -15,58 +16,57 @@ async function main() { thinkingConfig: { thinkingBudget: 2048, }, - }, + } as GoogleGenerativeAIProviderOptions, }, + temperature: 0, // Use temp 0 for this to make the model make better use of search grounding onError(error) { console.error(error); }, prompt: - 'Calculate 20th fibonacci number. Then find the nearest palindrome to it. Also, provide the current XMR to USD rate, provide sources.', + 'Calculate 20th fibonacci number. Then find the nearest palindrome to it. Also, provide the current XMR to USD rate.', }); for await (const delta of result.fullStream) { switch (delta.type) { case 'file': { if (delta.type === 'file') { - process.stdout.write( + console.log( '\x1b[33m' + delta.type + '\x1b[34m: ' + delta.file.mediaType + '\x1b[0m', ); - console.log(); console.log(atob(delta.file.base64 as string)); + console.log('\x1b[31m' + delta.type + '\x1b[0m'); } + break; } case 'text': { if (delta.type === 'text') { - process.stdout.write('\x1b[34m' + delta.type + '\x1b[0m'); - console.log(); console.log(delta.text); } + break; } case 'source': { if (delta.type === 'source' && delta.sourceType === 'url') { - process.stdout.write('\x1b[32m' + delta.type + '\x1b[0m'); - console.log(); console.log('ID:', delta.id); console.log('Title:', delta.title); console.log('URL:', delta.url); console.log(); } + break; } case 'reasoning': { if (delta.type === 'reasoning') { console.log('\x1b[34m' + delta.type); - console.log(); console.log(delta.text); console.log('\x1b[0m'); } + break; } case 'tool-call': { if (delta.type === 'tool-call') { - console.log('\x1b[33m' + delta.type); console.log( 'TOOL CALL: ', delta.toolName, @@ -77,10 +77,10 @@ async function main() { console.log('Args: ', delta.args); console.log('\x1b[0m'); } + break; } case 'tool-result': { if (delta.type === 'tool-result') { - console.log('\x1b[37m' + delta.type); console.log(); console.log( 'TOOL RESULT: ', @@ -92,37 +92,42 @@ async function main() { console.log(delta.result); console.log('\x1b[0m'); } + break; } case 'error': { - console.log('\x1b[31m' + delta.type + '\x1b[0m'); - console.log(); if (delta.type === 'error' && delta.error != null) { console.log(delta.error); } + break; } } } + console.log(); // Show sources - const metadata = (await result.providerMetadata) as - | GoogleGenerativeAIProviderMetadata - | undefined; - if (metadata != null) { - console.log('\x1b[31m' + 'sources' + '\x1b[0m'); - if ( - metadata?.groundingMetadata?.webSearchQueries && - metadata?.groundingMetadata?.webSearchQueries?.length > 0 - ) { - console.log( - 'Web Queries: ', - metadata?.groundingMetadata?.webSearchQueries, - ); - } - if (metadata.groundingMetadata?.searchEntryPoint != null) { - console.log( - 'Search Entry Point: ', - JSON.stringify(metadata.groundingMetadata?.searchEntryPoint, null, 2), - ); + const providerMetadata = await result.providerMetadata; + if ( + providerMetadata != null && + typeof providerMetadata === 'object' && + 'google' in providerMetadata && + providerMetadata.google != null + ) { + const metadata = + providerMetadata.google as unknown as GoogleGenerativeAIProviderMetadata; + if (metadata != null) { + console.log('\x1b[35m' + 'sources' + '\x1b[0m'); + if (metadata?.groundingMetadata?.webSearchQueries) { + console.log('\x1b[36m' + 'Web Queries:' + '\x1b[0m'); + for (const query of metadata?.groundingMetadata?.webSearchQueries) { + console.log(query); + } + } + if (metadata.groundingMetadata?.searchEntryPoint != null) { + console.log('\x1b[36m' + 'Search Entry Point: ' + '\x1b[0m'); + console.log( + JSON.stringify(metadata.groundingMetadata?.searchEntryPoint, null, 2), + ); + } } } console.log(); diff --git a/packages/google/src/google-generative-ai-language-model.ts b/packages/google/src/google-generative-ai-language-model.ts index 037e675e7479..8a0b13be5968 100644 --- a/packages/google/src/google-generative-ai-language-model.ts +++ b/packages/google/src/google-generative-ai-language-model.ts @@ -243,7 +243,7 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV2 { ) { content.push({ type: 'text' as const, - text: `Execution Result (Outcome: ${part.codeExecutionResult.outcome}):\n ${part.codeExecutionResult.output}`, + text: `Execution Result (Outcome: ${part.codeExecutionResult.outcome}):\n${part.codeExecutionResult.output}`, }); // function calls } else if ('functionCall' in part) { @@ -567,7 +567,7 @@ function getCodeExecutionResultStreamParts( // Ensure output might be empty but outcome is present const outputText = part.codeExecutionResult.output; // Only create a part if there's an outcome, even if output is empty. - if (part.codeExecutionResult.outcome.length > 0) { + if (part.codeExecutionResult != null) { const formattedText = `Execution Result (Outcome: ${part.codeExecutionResult.outcome}):\n${outputText}`; resultParts.push({ type: 'text', text: formattedText }); }