support google genai code execution tool

[altbdoor]

Description

Background

Google Gemini API supports code execution, which runs Python code in Google servers.

I was not able to set a codeExecution tool with the SDK, so I resorted to overriding the fetch.

import { createGoogleGenerativeAI } from "@ai-sdk/google";

const google = createGoogleGenerativeAI({
  apiKey: process.env.GOOGLE_API_KEY,
  fetch: async (url, options) => {
    const fixedOptions = JSON.parse(options!.body! as string);
    if (!('tools' in fixedOptions)) {
      fixedOptions['tools'] = [{ codeExecution: {} }]
    }

    options!.body = JSON.stringify(fixedOptions);
    return await fetch(url, options);
  },
});

Running streamText will then throw AI_TypeValidationError.

Show error

{
  type: 'error',
  error: _TypeValidationError [AI_TypeValidationError]: Type validation failed: Value: {"candidates":[{"content":{"parts":[{"codeExecutionResult":{"outcome":"OUTCOME_OK","output":"sum of first 50 primes = 4227\n"}}],"role":"model"},"index":0}],"usageMetadata":{"promptTokenCount":659,"candidatesTokenCount":74,"totalTokenCount":733}}.
  Error message: [
    {
      "code": "invalid_union",
      "unionErrors": [
        {
          "issues": [
            {
              "code": "invalid_type",
              "expected": "string",
              "received": "undefined",
              "path": [
                "candidates",
                0,
                "content",
                "parts",
                0,
                "text"
              ],
              "message": "Required"
            }
          ],
          "name": "ZodError"
        },
        {
          "issues": [
            {
              "code": "invalid_type",
              "expected": "object",
              "received": "undefined",
              "path": [
                "candidates",
                0,
                "content",
                "parts",
                0,
                "functionCall"
              ],
              "message": "Required"
            }
          ],
          "name": "ZodError"
        }
      ],
      "path": [
        "candidates",
        0,
        "content",
        "parts",
        0
      ],
      "message": "Invalid input"
    }
  ]
      at _TypeValidationError.wrap (webpack-internal:///(action-browser)/./node_modules/@ai-sdk/provider/dist/index.mjs:483:86)
      at safeValidateTypes (webpack-internal:///(action-browser)/./node_modules/@ai-sdk/provider-utils/dist/index.mjs:257:80)
      at safeParseJSON (webpack-internal:///(action-browser)/./node_modules/@ai-sdk/provider-utils/dist/index.mjs:297:12)
      ... 45 lines matching cause stack trace ...
      at readableByteStreamControllerEnqueue (node:internal/webstreams/readablestream:2837:7)
      at ReadableByteStreamController.enqueue (node:internal/webstreams/readablestream:1184:5) {
    cause: ZodError: [
      {
        "code": "invalid_union",
        "unionErrors": [
          {
            "issues": [
              {
                "code": "invalid_type",
                "expected": "string",
                "received": "undefined",
                "path": [
                  "candidates",
                  0,
                  "content",
                  "parts",
                  0,
                  "text"
                ],
                "message": "Required"
              }
            ],
            "name": "ZodError"
          },
          {
            "issues": [
              {
                "code": "invalid_type",
                "expected": "object",
                "received": "undefined",
                "path": [
                  "candidates",
                  0,
                  "content",
                  "parts",
                  0,
                  "functionCall"
                ],
                "message": "Required"
              }
            ],
            "name": "ZodError"
          }
        ],
        "path": [
          "candidates",
          0,
          "content",
          "parts",
          0
        ],
        "message": "Invalid input"
      }
    ]
        at get error (webpack-internal:///(action-browser)/./node_modules/zod/lib/index.mjs:699:31)
        at Object.eval [as validate] (webpack-internal:///(action-browser)/./node_modules/@ai-sdk/provider-utils/dist/index.mjs:227:101)
        at safeValidateTypes (webpack-internal:///(action-browser)/./node_modules/@ai-sdk/provider-utils/dist/index.mjs:251:31)
        at safeParseJSON (webpack-internal:///(action-browser)/./node_modules/@ai-sdk/provider-utils/dist/index.mjs:297:12)
        at Object.transform (webpack-internal:///(action-browser)/./node_modules/@ai-sdk/provider-utils/dist/index.mjs:503:13)
        at invokePromiseCallback (node:internal/webstreams/util:181:10)
        at Object.transformAlgorithm (node:internal/webstreams/util:186:23)
        at transformStreamDefaultControllerPerformTransform (node:internal/webstreams/transformstream:526:37)
        at transformStreamDefaultSinkWriteAlgorithm (node:internal/webstreams/transformstream:572:10)
        at node:internal/webstreams/transformstream:377:16
        at writableStreamDefaultControllerProcessWrite (node:internal/webstreams/writablestream:1127:5)
        at writableStreamDefaultControllerAdvanceQueueIfNeeded (node:internal/webstreams/writablestream:1242:5)
        at writableStreamDefaultControllerWrite (node:internal/webstreams/writablestream:1116:3)
        at writableStreamDefaultWriterWrite (node:internal/webstreams/writablestream:1006:3)
        at [kChunk] (node:internal/webstreams/readablestream:1585:31)
        at readableStreamFulfillReadRequest (node:internal/webstreams/readablestream:2118:24)
        at readableStreamDefaultControllerEnqueue (node:internal/webstreams/readablestream:2310:5)
        at transformStreamDefaultControllerEnqueue (node:internal/webstreams/transformstream:507:5)
        at TransformStreamDefaultController.enqueue (node:internal/webstreams/transformstream:323:5)
        at eval (webpack-internal:///(action-browser)/./node_modules/eventsource-parser/dist/stream.js:14:24)
        at parseEventStreamLine (webpack-internal:///(action-browser)/./node_modules/eventsource-parser/dist/index.js:77:9)
        at Object.feed (webpack-internal:///(action-browser)/./node_modules/eventsource-parser/dist/index.js:65:7)
        at Object.transform (webpack-internal:///(action-browser)/./node_modules/eventsource-parser/dist/stream.js:19:16)
        at invokePromiseCallback (node:internal/webstreams/util:181:10)
        at Object.transformAlgorithm (node:internal/webstreams/util:186:23)
        at transformStreamDefaultControllerPerformTransform (node:internal/webstreams/transformstream:526:37)
        at transformStreamDefaultSinkWriteAlgorithm (node:internal/webstreams/transformstream:572:10)
        at node:internal/webstreams/transformstream:377:16
        at writableStreamDefaultControllerProcessWrite (node:internal/webstreams/writablestream:1127:5)
        at writableStreamDefaultControllerAdvanceQueueIfNeeded (node:internal/webstreams/writablestream:1242:5)
        at writableStreamDefaultControllerWrite (node:internal/webstreams/writablestream:1116:3)
        at writableStreamDefaultWriterWrite (node:internal/webstreams/writablestream:1006:3)
        at [kChunk] (node:internal/webstreams/readablestream:1585:31)
        at readableStreamFulfillReadRequest (node:internal/webstreams/readablestream:2118:24)
        at readableStreamDefaultControllerEnqueue (node:internal/webstreams/readablestream:2310:5)
        at transformStreamDefaultControllerEnqueue (node:internal/webstreams/transformstream:507:5)
        at TransformStreamDefaultController.enqueue (node:internal/webstreams/transformstream:323:5)
        at Object.transform (node:internal/webstreams/encoding:138:22)
        at invokePromiseCallback (node:internal/webstreams/util:181:10)
        at Object.transformAlgorithm (node:internal/webstreams/util:186:23)
        at transformStreamDefaultControllerPerformTransform (node:internal/webstreams/transformstream:526:37)
        at transformStreamDefaultSinkWriteAlgorithm (node:internal/webstreams/transformstream:572:10)
        at node:internal/webstreams/transformstream:377:16
        at writableStreamDefaultControllerProcessWrite (node:internal/webstreams/writablestream:1127:5)
        at writableStreamDefaultControllerAdvanceQueueIfNeeded (node:internal/webstreams/writablestream:1242:5)
        at writableStreamDefaultControllerWrite (node:internal/webstreams/writablestream:1116:3)
        at writableStreamDefaultWriterWrite (node:internal/webstreams/writablestream:1006:3)
        at [kChunk] (node:internal/webstreams/readablestream:1585:31)
        at readableStreamFulfillReadRequest (node:internal/webstreams/readablestream:2118:24)
        at readableByteStreamControllerEnqueue (node:internal/webstreams/readablestream:2837:7) {
      issues: [Array],
      addIssue: [Function (anonymous)],
      addIssues: [Function (anonymous)],
      errors: [Array]
    },
    value: { candidates: [Array], usageMetadata: [Object] },
    [Symbol(vercel.ai.error)]: true,
    [Symbol(vercel.ai.error.AI_TypeValidationError)]: true
  }
}

I tried logging the chunk response, and it appears that the chunk does not have the text property.

Show chunk response

// first chunk
{
  "candidates": [
    {
      "content": {
        "parts": [
          {
            "executableCode": {
              "language": "PYTHON",
              "code": "\nimport math\n\ndef is_prime(num):\n    \"\"\"\n    Checks if a number is prime.\n    \"\"\"\n    if num <= 1:\n        return False\n    for i in range(2, int(math.sqrt(num)) + 1):\n        if num % i == 0:\n            return False\n    return True\n\nprimes = []\nnum = 2\ncount = 0\nwhile count < 50:\n    if is_prime(num):\n        primes.append(num)\n        count += 1\n    num += 1\n\nprint(f'The sum of the first 50 prime numbers is: {sum(primes)}')\n"
            }
          }
        ],
        "role": "model"
      },
      "finishReason": "STOP",
      "index": 0,
      "safetyRatings": [
        {
          "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
          "probability": "NEGLIGIBLE"
        },
        {
          "category": "HARM_CATEGORY_HATE_SPEECH",
          "probability": "NEGLIGIBLE"
        },
        {
          "category": "HARM_CATEGORY_HARASSMENT",
          "probability": "NEGLIGIBLE"
        },
        {
          "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
          "probability": "NEGLIGIBLE"
        }
      ]
    }
  ],
  "usageMetadata": {
    "promptTokenCount": 659,
    "candidatesTokenCount": 197,
    "totalTokenCount": 856
  }
}

// second chunk
{
  "candidates": [
    {
      "content": {
        "parts": [
          {
            "codeExecutionResult": {
              "outcome": "OUTCOME_OK",
              "output": "The sum of the first 50 prime numbers is: 5117\n"
            }
          }
        ],
        "role": "model"
      },
      "index": 0
    }
  ],
  "usageMetadata": {
    "promptTokenCount": 659,
    "candidatesTokenCount": 197,
    "totalTokenCount": 856
  }
}

This format appears to be enforced in the schema, defined in:

ai/packages/google/src/google-generative-ai-language-model.ts

Lines 389 to 404 in 14eb2b8

	const contentSchema = z.object({
	role: z.string(),
	parts: z.array(
	z.union([
	z.object({
	text: z.string(),
	}),
	z.object({
	functionCall: z.object({
	name: z.string(),
	args: z.unknown(),
	}),
	}),
	]),
	),
	});

Question

So I have a number of questions here:

1. Should the library allow devs to set the codeExecution tool for Gemini somehow?
2. Should the library adapt to the two code execution parts (executableCode and codeExecutionResult)?

Workaround

If I access the textStream, the code will throw an error:

const { textStream, fullStream } = await streamText({
  model: google('gemini-1.5-flash-latest'),
  /* other configs */
});

// this throws an error as the zod validation fails
for await (const chunk of textStream) {
  stream.update(chunk);
}

So the work around I had was to access the fullStream, and silently ignore these errors.

const { textStream, fullStream } = await streamText({
  model: google('gemini-1.5-flash-latest'),
  /* other configs */
});

for await (const chunk of fullStream) {
  if (chunk.type === "text-delta" && !!chunk.textDelta) {
    stream.update(chunk.textDelta);
  } else if (
    chunk.type === 'error' &&
    chunk.error instanceof TypeValidationError
  ) {
    // silently ignore
    // maybe better conditions can be set to really identify the zod validation error
  }
}

Code example

No response

Additional context

Packages installed:

$ npm ls
<redacted>@0.1.0 <redacted>
├── @ai-sdk/google@0.0.51
├── @ai-sdk/openai@0.0.63
├── @types/dompurify@3.0.5
├── @types/node@20.16.10
├── @types/react-dom@18.3.0
├── @types/react@18.3.10
├── @vercel/blob@0.24.0
├── ai@3.4.7
├── dompurify@3.1.7
├── eslint-config-next@14.2.14
├── eslint@8.57.1
├── highlight.js@11.10.0
├── marked@14.1.2
├── next-auth@5.0.0-beta.22
├── next@14.2.14
├── prettier@3.3.3
├── react-dom@18.3.1
├── react@18.3.1
├── typescript@5.6.2
└── zod@3.23.8

[luqman-sazali-ad]

I am encountering the same issue where TypeValidationError.isInstance(chunk.error) returns false, even though the error is displayed as _TypeValidationError. However, it returns true when using AISDKError.isInstance(chunk.error). Is this the expected behavior?

[westn]

Thanks for this thread! I also encountered an issue running code execution.
Solved it with the following fetch (a bit hacky)

createGoogleGenerativeAI({
      apiKey: this.apiKey,
      fetch: async (url, options) => {
        const body = JSON.parse(options.body);
        let tools: undefined | object;
        if ('tools' in body) {
          tools = [{ codeExecution: {} }];
        }
        const sendBody = JSON.stringify({
          ...body,
          tools,
        });
        return fetch(url, {
          ...options,
          body: sendBody,
        }).then(async (res) => {
          const clonedRes = res.clone();
          const resBody = await clonedRes.json();

          if (resBody.candidates) {
            resBody.candidates = resBody.candidates.map((candidate) => {
              if (candidate.content && candidate.content.parts) {
                return {
                  ...candidate,
                  content: {
                    ...candidate.content,
                    parts: candidate.content.parts.filter(
                      (part) =>
                        !part.executableCode && !part.codeExecutionResult,
                    ),
                  },
                };
              }
              return candidate;
            });
          }

          return new Response(JSON.stringify(resBody), {
            status: res.status,
            statusText: res.statusText,
            headers: res.headers,
          });
        });
      },
    })('gemini-2.0-flash-thinking-exp-01-21');