From 0fd5e2d70da3d54b034e1410e9b598085bad03a6 Mon Sep 17 00:00:00 2001 From: Tina Nguyen Date: Sat, 11 Apr 2026 01:26:22 -0400 Subject: [PATCH 1/9] wip --- plugins/google/src/beta/realtime/realtime_api.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index 54cd9a920..f5c466e91 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -547,13 +547,6 @@ export class RealtimeSession extends llm.RealtimeSession { } async updateInstructions(instructions: string): Promise { - if (this.options.model === 'gemini-3.1-flash-live-preview') { - this.#logger.warn( - 'updateInstructions is not compatible with gemini-3.1-flash-live-preview and will be ignored.', - ); - this.options.instructions = instructions; - return; - } if (this.options.instructions === undefined || this.options.instructions !== instructions) { this.options.instructions = instructions; this.markRestartNeeded(); @@ -561,13 +554,6 @@ export class RealtimeSession extends llm.RealtimeSession { } async updateChatCtx(chatCtx: llm.ChatContext): Promise { - if (this.options.model === 'gemini-3.1-flash-live-preview') { - this.#logger.warn( - 'updateChatCtx is not compatible with gemini-3.1-flash-live-preview and will be ignored.', - ); - this._chatCtx = chatCtx.copy(); - return; - } const unlock = await this.sessionLock.lock(); try { if (!this.activeSession) { From 876c983ca57f6c072b23156e8f90e3a475cca368 Mon Sep 17 00:00:00 2001 From: Tina Nguyen Date: Mon, 13 Apr 2026 14:25:28 -0400 Subject: [PATCH 2/9] fix --- plugins/google/src/beta/realtime/api_proto.ts | 1 - .../google/src/beta/realtime/realtime_api.ts | 104 ++++++++++++------ 2 files changed, 68 insertions(+), 37 deletions(-) diff --git a/plugins/google/src/beta/realtime/api_proto.ts b/plugins/google/src/beta/realtime/api_proto.ts index c35ff9570..c1099d1ad 100644 --- a/plugins/google/src/beta/realtime/api_proto.ts +++ b/plugins/google/src/beta/realtime/api_proto.ts @@ -21,7 +21,6 @@ export type LiveAPIModels = // Gemini API models | 'gemini-3.1-flash-live-preview' // https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-live-preview | 'gemini-2.5-flash-native-audio-preview-12-2025' // https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-live - | 'gemini-2.5-flash-native-audio-preview-09-2025' // https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-live | 'gemini-2.0-flash-exp'; // still works in Gemini API but not VertexAI /** diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index f5c466e91..a8c5c0102 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -303,6 +303,19 @@ export class RealtimeModel extends llm.RealtimeModel { if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) { serverTurnDetection = false; } + // Environment variable fallbacks + const apiKey = options.apiKey || process.env.GOOGLE_API_KEY; + const project = options.project || process.env.GOOGLE_CLOUD_PROJECT; + const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1'; + const vertexai = options.vertexai ?? false; + + // Model selection based on API type + const defaultModel = vertexai + ? 'gemini-live-2.5-flash-native-audio' + : 'gemini-2.5-flash-native-audio-preview-12-2025'; + + const model = options.model || defaultModel; + const mutableSession = !model.includes('3.1'); super({ messageTruncation: false, @@ -311,25 +324,14 @@ export class RealtimeModel extends llm.RealtimeModel { autoToolReplyGeneration: true, audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true, manualFunctionCalls: false, - midSessionChatCtxUpdate: false, - midSessionInstructionsUpdate: true, + midSessionChatCtxUpdate: mutableSession, + midSessionInstructionsUpdate: mutableSession, midSessionToolsUpdate: false, perResponseToolChoice: false, }); - // Environment variable fallbacks - const apiKey = options.apiKey || process.env.GOOGLE_API_KEY; - const project = options.project || process.env.GOOGLE_CLOUD_PROJECT; - const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1'; - const vertexai = options.vertexai ?? false; - - // Model selection based on API type - const defaultModel = vertexai - ? 'gemini-live-2.5-flash-native-audio' - : 'gemini-2.5-flash-native-audio-preview-12-2025'; - this._options = { - model: options.model || defaultModel, + model, apiKey, voice: options.voice || 'Puck', language: options.language ? normalizeLanguage(options.language) : undefined, @@ -547,10 +549,41 @@ export class RealtimeSession extends llm.RealtimeSession { } async updateInstructions(instructions: string): Promise { - if (this.options.instructions === undefined || this.options.instructions !== instructions) { - this.options.instructions = instructions; - this.markRestartNeeded(); + if (this.options.instructions !== undefined && this.options.instructions === instructions) { + return; + } + + this.options.instructions = instructions; + + const unlock = await this.sessionLock.lock(); + try { + if (!this.activeSession) { + this.markRestartNeeded(); + return; + } + } finally { + unlock(); + } + + if (!this.realtimeModel.capabilities.midSessionInstructionsUpdate) { + return; } + + this.#logger.debug('Updating instructions mid-session'); + this.sendClientEvent({ + type: 'content', + value: { + turns: [ + { + parts: [{ text: instructions }], + // Vertex AI ignores role=None or role="system" and only works with role="model". + // Gemini Live API (non-Vertex) errors on role="system"; role=None works as system role. + role: this.options.vertexai ? 'model' : undefined, + }, + ], + turnComplete: false, + }, + }); } async updateChatCtx(chatCtx: llm.ChatContext): Promise { @@ -609,20 +642,21 @@ export class RealtimeSession extends llm.RealtimeSession { } } - this.sendClientEvent({ - type: 'content', - value: { - turns: turns as types.Content[], - turnComplete: false, - }, - }); - } - - if (toolResults) { - this.sendClientEvent({ - type: 'tool_response', - value: toolResults, - }); + if (toolResults) { + this.sendClientEvent({ + type: 'tool_response', + value: toolResults, + }); + } + if (this.realtimeModel.capabilities.midSessionChatCtxUpdate) { + this.sendClientEvent({ + type: 'content', + value: { + turns: turns as types.Content[], + turnComplete: false, + }, + }); + } } } @@ -686,13 +720,11 @@ export class RealtimeSession extends llm.RealtimeSession { } async generateReply(instructions?: string): Promise { - if (this.options.model === 'gemini-3.1-flash-live-preview') { + if (!this.realtimeModel.capabilities.midSessionChatCtxUpdate) { this.#logger.warn( - 'generateReply is not compatible with gemini-3.1-flash-live-preview. Use a Gemini 2.5 live model for voice-agent flows that require programmatic reply generation.', - ); - throw new Error( - "generateReply is not compatible with 'gemini-3.1-flash-live-preview'; use a Gemini 2.5 live model for voice-agent flows that require programmatic reply generation.", + `generateReply is not compatible with '${this.options.model}' and will be ignored.`, ); + throw new Error(`generateReply is not compatible with '${this.options.model}'`); } if (this.pendingGenerationFut && !this.pendingGenerationFut.done) { From 93f6a648b08c4b69c423d574206f24d9cc979692 Mon Sep 17 00:00:00 2001 From: Tina Nguyen Date: Mon, 13 Apr 2026 14:27:10 -0400 Subject: [PATCH 3/9] changeset --- .changeset/cruel-deer-brake.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cruel-deer-brake.md diff --git a/.changeset/cruel-deer-brake.md b/.changeset/cruel-deer-brake.md new file mode 100644 index 000000000..c457408dc --- /dev/null +++ b/.changeset/cruel-deer-brake.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-google': patch +--- + +(gemini live 3.1): send tool responses and block sendClientContent where applicable. also remove deprecated gemini model From 3235c184f220c3cfa69759c2614117197fde6d2b Mon Sep 17 00:00:00 2001 From: Tina Nguyen Date: Mon, 13 Apr 2026 14:34:00 -0400 Subject: [PATCH 4/9] add init warning about 3.1 limitations --- plugins/google/package.json | 2 +- plugins/google/src/beta/realtime/realtime_api.ts | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/google/package.json b/plugins/google/package.json index 2bdc2f165..e7bca6097 100644 --- a/plugins/google/package.json +++ b/plugins/google/package.json @@ -43,7 +43,7 @@ "typescript": "^5.0.0" }, "dependencies": { - "@google/genai": "^1.44.0", + "@google/genai": "^1.49.0", "@livekit/mutex": "^1.1.1", "@types/json-schema": "^7.0.15", "json-schema": "^0.4.0" diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index a8c5c0102..8eddd38c0 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -140,6 +140,7 @@ interface ResponseGeneration { * Google Realtime Model for real-time voice conversations with Gemini models */ export class RealtimeModel extends llm.RealtimeModel { + #logger = log(); /** @internal */ _options: RealtimeOptions; @@ -330,6 +331,12 @@ export class RealtimeModel extends llm.RealtimeModel { perResponseToolChoice: false, }); + if (!mutableSession) { + this.#logger.warn( + `'${model}' has limited mid-session update support. instructions, chat context, and tool updates will not be applied until the next session.`, + ); + } + this._options = { model, apiKey, From 32afe91f33696dfed65118944eac189592b6dd0e Mon Sep 17 00:00:00 2001 From: Tina Nguyen Date: Mon, 13 Apr 2026 14:36:16 -0400 Subject: [PATCH 5/9] add pnpmlock --- pnpm-lock.yaml | 62 +++++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 42f4ed282..a8f4c993c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -553,8 +553,8 @@ importers: plugins/google: dependencies: '@google/genai': - specifier: ^1.44.0 - version: 1.44.0 + specifier: ^1.49.0 + version: 1.49.0 '@livekit/mutex': specifier: ^1.1.1 version: 1.1.1 @@ -668,6 +668,31 @@ importers: specifier: ^5.0.0 version: 5.9.3 + plugins/liveavatar: + dependencies: + livekit-server-sdk: + specifier: ^2.13.3 + version: 2.14.1 + devDependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/rtc-node': + specifier: ^0.13.22 + version: 0.13.25 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@22.19.1) + pino: + specifier: ^8.19.0 + version: 8.21.0 + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.5.6)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.0.0 + version: 5.9.3 + plugins/livekit: dependencies: '@huggingface/hub': @@ -1804,8 +1829,8 @@ packages: cpu: [x64] os: [win32] - '@google/genai@1.44.0': - resolution: {integrity: sha512-kRt9ZtuXmz+tLlcNntN/VV4LRdpl6ZOu5B1KbfNgfR65db15O6sUQcwnwLka8sT/V6qysD93fWrgJHF2L7dA9A==} + '@google/genai@1.49.0': + resolution: {integrity: sha512-hO69Zl0H3x+L0KL4stl1pLYgnqnwHoLqtKy6MRlNnW8TAxjqMdOUVafomKd4z1BePkzoxJWbYILny9a2Zk43VQ==} engines: {node: '>=20.0.0'} peerDependencies: '@modelcontextprotocol/sdk': ^1.25.2 @@ -4619,10 +4644,6 @@ packages: prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} - protobufjs@7.4.0: - resolution: {integrity: sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==} - engines: {node: '>=12.0.0'} - protobufjs@7.5.4: resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==} engines: {node: '>=12.0.0'} @@ -6149,12 +6170,12 @@ snapshots: '@ffmpeg-installer/win32-x64@4.1.0': optional: true - '@google/genai@1.44.0': + '@google/genai@1.49.0': dependencies: google-auth-library: 10.5.0 p-retry: 4.6.2 protobufjs: 7.5.4 - ws: 8.18.3 + ws: 8.20.0 transitivePeerDependencies: - bufferutil - supports-color @@ -6616,7 +6637,7 @@ snapshots: '@opentelemetry/sdk-logs': 0.208.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': 2.2.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 2.2.0(@opentelemetry/api@1.9.0) - protobufjs: 7.4.0 + protobufjs: 7.5.4 '@opentelemetry/otlp-transformer@0.54.2(@opentelemetry/api@1.9.0)': dependencies: @@ -6627,7 +6648,7 @@ snapshots: '@opentelemetry/sdk-logs': 0.54.2(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': 1.27.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.27.0(@opentelemetry/api@1.9.0) - protobufjs: 7.4.0 + protobufjs: 7.5.4 '@opentelemetry/propagator-b3@1.30.1(@opentelemetry/api@1.9.0)': dependencies: @@ -9028,7 +9049,7 @@ snapshots: long: 5.2.3 onnxruntime-common: 1.22.0-dev.20250409-89f8206ba4 platform: 1.3.6 - protobufjs: 7.4.0 + protobufjs: 7.5.4 openai@6.8.1(ws@8.18.3)(zod@3.25.76): optionalDependencies: @@ -9285,21 +9306,6 @@ snapshots: object-assign: 4.1.1 react-is: 16.13.1 - protobufjs@7.4.0: - dependencies: - '@protobufjs/aspromise': 1.1.2 - '@protobufjs/base64': 1.1.2 - '@protobufjs/codegen': 2.0.4 - '@protobufjs/eventemitter': 1.1.0 - '@protobufjs/fetch': 1.1.0 - '@protobufjs/float': 1.0.2 - '@protobufjs/inquire': 1.1.0 - '@protobufjs/path': 1.1.2 - '@protobufjs/pool': 1.1.0 - '@protobufjs/utf8': 1.1.0 - '@types/node': 22.19.1 - long: 5.2.3 - protobufjs@7.5.4: dependencies: '@protobufjs/aspromise': 1.1.2 From a8c332143beaa2faf2d9e4af922258d95438418a Mon Sep 17 00:00:00 2001 From: Tina Nguyen Date: Mon, 13 Apr 2026 14:39:04 -0400 Subject: [PATCH 6/9] rm noise --- pnpm-lock.yaml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a8f4c993c..a1137bcbd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -668,31 +668,6 @@ importers: specifier: ^5.0.0 version: 5.9.3 - plugins/liveavatar: - dependencies: - livekit-server-sdk: - specifier: ^2.13.3 - version: 2.14.1 - devDependencies: - '@livekit/agents': - specifier: workspace:* - version: link:../../agents - '@livekit/rtc-node': - specifier: ^0.13.22 - version: 0.13.25 - '@microsoft/api-extractor': - specifier: ^7.35.0 - version: 7.43.7(@types/node@22.19.1) - pino: - specifier: ^8.19.0 - version: 8.21.0 - tsup: - specifier: ^8.3.5 - version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.19.1))(postcss@8.5.6)(tsx@4.21.0)(typescript@5.9.3) - typescript: - specifier: ^5.0.0 - version: 5.9.3 - plugins/livekit: dependencies: '@huggingface/hub': From 47b8b0fada566f08acfc7cd273cb7a250b34936e Mon Sep 17 00:00:00 2001 From: Tina Nguyen Date: Mon, 13 Apr 2026 14:42:52 -0400 Subject: [PATCH 7/9] undo package bump --- plugins/google/package.json | 2 +- pnpm-lock.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/google/package.json b/plugins/google/package.json index e7bca6097..2bdc2f165 100644 --- a/plugins/google/package.json +++ b/plugins/google/package.json @@ -43,7 +43,7 @@ "typescript": "^5.0.0" }, "dependencies": { - "@google/genai": "^1.49.0", + "@google/genai": "^1.44.0", "@livekit/mutex": "^1.1.1", "@types/json-schema": "^7.0.15", "json-schema": "^0.4.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a1137bcbd..8563dbe9d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -553,7 +553,7 @@ importers: plugins/google: dependencies: '@google/genai': - specifier: ^1.49.0 + specifier: ^1.44.0 version: 1.49.0 '@livekit/mutex': specifier: ^1.1.1 From 1fb14bd3d687b702c8e5067c2d4685d9f4586ebb Mon Sep 17 00:00:00 2001 From: Tina Nguyen Date: Mon, 13 Apr 2026 14:45:08 -0400 Subject: [PATCH 8/9] fix --- pnpm-lock.yaml | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8563dbe9d..42f4ed282 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -554,7 +554,7 @@ importers: dependencies: '@google/genai': specifier: ^1.44.0 - version: 1.49.0 + version: 1.44.0 '@livekit/mutex': specifier: ^1.1.1 version: 1.1.1 @@ -1804,8 +1804,8 @@ packages: cpu: [x64] os: [win32] - '@google/genai@1.49.0': - resolution: {integrity: sha512-hO69Zl0H3x+L0KL4stl1pLYgnqnwHoLqtKy6MRlNnW8TAxjqMdOUVafomKd4z1BePkzoxJWbYILny9a2Zk43VQ==} + '@google/genai@1.44.0': + resolution: {integrity: sha512-kRt9ZtuXmz+tLlcNntN/VV4LRdpl6ZOu5B1KbfNgfR65db15O6sUQcwnwLka8sT/V6qysD93fWrgJHF2L7dA9A==} engines: {node: '>=20.0.0'} peerDependencies: '@modelcontextprotocol/sdk': ^1.25.2 @@ -4619,6 +4619,10 @@ packages: prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} + protobufjs@7.4.0: + resolution: {integrity: sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==} + engines: {node: '>=12.0.0'} + protobufjs@7.5.4: resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==} engines: {node: '>=12.0.0'} @@ -6145,12 +6149,12 @@ snapshots: '@ffmpeg-installer/win32-x64@4.1.0': optional: true - '@google/genai@1.49.0': + '@google/genai@1.44.0': dependencies: google-auth-library: 10.5.0 p-retry: 4.6.2 protobufjs: 7.5.4 - ws: 8.20.0 + ws: 8.18.3 transitivePeerDependencies: - bufferutil - supports-color @@ -6612,7 +6616,7 @@ snapshots: '@opentelemetry/sdk-logs': 0.208.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': 2.2.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 2.2.0(@opentelemetry/api@1.9.0) - protobufjs: 7.5.4 + protobufjs: 7.4.0 '@opentelemetry/otlp-transformer@0.54.2(@opentelemetry/api@1.9.0)': dependencies: @@ -6623,7 +6627,7 @@ snapshots: '@opentelemetry/sdk-logs': 0.54.2(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': 1.27.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.27.0(@opentelemetry/api@1.9.0) - protobufjs: 7.5.4 + protobufjs: 7.4.0 '@opentelemetry/propagator-b3@1.30.1(@opentelemetry/api@1.9.0)': dependencies: @@ -9024,7 +9028,7 @@ snapshots: long: 5.2.3 onnxruntime-common: 1.22.0-dev.20250409-89f8206ba4 platform: 1.3.6 - protobufjs: 7.5.4 + protobufjs: 7.4.0 openai@6.8.1(ws@8.18.3)(zod@3.25.76): optionalDependencies: @@ -9281,6 +9285,21 @@ snapshots: object-assign: 4.1.1 react-is: 16.13.1 + protobufjs@7.4.0: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 22.19.1 + long: 5.2.3 + protobufjs@7.5.4: dependencies: '@protobufjs/aspromise': 1.1.2 From 273089e5887ea4d09a08c0a977d35485935e0fd5 Mon Sep 17 00:00:00 2001 From: Tina Nguyen Date: Mon, 13 Apr 2026 18:47:27 -0400 Subject: [PATCH 9/9] fix --- plugins/google/src/beta/realtime/realtime_api.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index 8eddd38c0..2f9caf087 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -649,12 +649,6 @@ export class RealtimeSession extends llm.RealtimeSession { } } - if (toolResults) { - this.sendClientEvent({ - type: 'tool_response', - value: toolResults, - }); - } if (this.realtimeModel.capabilities.midSessionChatCtxUpdate) { this.sendClientEvent({ type: 'content', @@ -665,6 +659,13 @@ export class RealtimeSession extends llm.RealtimeSession { }); } } + + if (toolResults) { + this.sendClientEvent({ + type: 'tool_response', + value: toolResults, + }); + } } // since we don't have a view of the history on the server side, we'll assume