From 96d99f2ec3cb41803cf952e664d4e90db0657959 Mon Sep 17 00:00:00 2001 From: Qiong Zhou Huang Date: Tue, 21 Apr 2026 18:44:37 -0700 Subject: [PATCH 1/8] fix(phonic): include tool calls and outputs in reset turn history buildTurnHistory previously only serialized text messages, dropping tool calls and their outputs from the system prompt sent on mid-session reset. Agents handed off mid-session lost access to any state that only existed in tool return values. Unify both history-building sites (pre-config postfix and reset) on a shared chatItemToText helper emitting simple XML-ish tags. Tool outputs are sliced to 16k chars to bound size. --- plugins/phonic/src/realtime/realtime_model.ts | 47 ++++++++++--------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/plugins/phonic/src/realtime/realtime_model.ts b/plugins/phonic/src/realtime/realtime_model.ts index a966e06f6..3b17e67e7 100644 --- a/plugins/phonic/src/realtime/realtime_model.ts +++ b/plugins/phonic/src/realtime/realtime_model.ts @@ -24,6 +24,7 @@ const PHONIC_INPUT_FRAME_MS = 20; const DEFAULT_MODEL = 'merritt'; const WS_CLOSE_NORMAL = 1000; const TOOL_CALL_OUTPUT_TIMEOUT_MS = 60_000; +const TOOL_CALL_OUTPUT_MAX_CHARS_FOR_HISTORY = 16_000; const CONVERSATION_HISTORY_PREFIX = '\n\nThis conversation is being continued from an existing ' + 'conversation. You are the assistant speaking to the user. ' + @@ -301,17 +302,8 @@ export class RealtimeSession extends llm.RealtimeSession { async updateChatCtx(chatCtx: llm.ChatContext): Promise { if (!this.configSent) { if (chatCtx.items.length > 0) { - const turnHistory = chatCtx.items - .filter( - (item): item is llm.ChatMessage => - item.type === 'message' && - 'textContent' in item && - item.textContent !== undefined && - item.textContent.trim() !== '', - ) - .map((item) => `${item.role}: ${item.textContent}`) - .join('\n'); - if (turnHistory.trim() !== '') { + const turnHistory = this.buildTurnHistory(chatCtx); + if (turnHistory) { this.#logger.debug( 'updateChatCtx called with messages prior to config being sent to Phonic. Including conversation state in system instructions.', ); @@ -448,6 +440,7 @@ export class RealtimeSession extends llm.RealtimeSession { if (this.socket) { this.#logger.info('Sending mid-session reset to Phonic'); + this.#logger.debug({ systemPrompt }, 'Phonic reset system prompt'); this.socket.sendReset({ type: 'reset', config: this.buildConfigOptions({ systemPrompt, toolsPayload }), @@ -894,16 +887,13 @@ export class RealtimeSession extends llm.RealtimeSession { } private buildTurnHistory(chatCtx: llm.ChatContext): string | undefined { - const messages = chatCtx.items.filter( - (item): item is llm.ChatMessage => - item.type === 'message' && - 'textContent' in item && - item.textContent !== undefined && - item.textContent.trim() !== '', - ); - if (messages.length === 0) return undefined; - const history = messages.map((m) => `${m.role}: ${m.textContent}`).join('\n'); - return history.trim() || undefined; + const lines: string[] = []; + for (const item of chatCtx.items) { + const text = chatItemToText(item); + if (text) lines.push(text); + } + if (lines.length === 0) return undefined; + return lines.join('\n'); } private *resampleAudio(frame: AudioFrame): Generator { @@ -936,3 +926,18 @@ export class RealtimeSession extends llm.RealtimeSession { } } } + +function chatItemToText(item: llm.ChatItem): string | undefined { + if (item.type === 'message') { + if (item.textContent === undefined) return undefined; + return `<${item.role}>${item.textContent}`; + } + if (item.type === 'function_call') { + return `${item.args}`; + } + if (item.type === 'function_call_output') { + const tag = item.isError ? 'tool_error' : 'tool_output'; + return `<${tag} name="${item.name}">${item.output.slice(0, TOOL_CALL_OUTPUT_MAX_CHARS_FOR_HISTORY)}`; + } + return undefined; +} From 027a343be56703f7ff3268d0b15e6f800bd2b081 Mon Sep 17 00:00:00 2001 From: Qiong Zhou Huang Date: Tue, 21 Apr 2026 19:03:48 -0700 Subject: [PATCH 2/8] filter out empty turns --- plugins/phonic/src/realtime/realtime_model.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/phonic/src/realtime/realtime_model.ts b/plugins/phonic/src/realtime/realtime_model.ts index 3b17e67e7..abab2859c 100644 --- a/plugins/phonic/src/realtime/realtime_model.ts +++ b/plugins/phonic/src/realtime/realtime_model.ts @@ -929,8 +929,9 @@ export class RealtimeSession extends llm.RealtimeSession { function chatItemToText(item: llm.ChatItem): string | undefined { if (item.type === 'message') { - if (item.textContent === undefined) return undefined; - return `<${item.role}>${item.textContent}`; + const text = item.textContent?.trim(); + if (!text) return undefined; + return `<${item.role}>${text}`; } if (item.type === 'function_call') { return `${item.args}`; From d05c050e198e3e5c8fcc23fd6b9c0fe73bd004aa Mon Sep 17 00:00:00 2001 From: Qiong Zhou Huang Date: Tue, 21 Apr 2026 19:21:01 -0700 Subject: [PATCH 3/8] comment out exclude function call --- agents/src/voice/agent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index ab7675523..5bde87b95 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -648,7 +648,7 @@ export class AgentTask extends Agent Date: Wed, 22 Apr 2026 13:57:01 -0700 Subject: [PATCH 4/8] update --- agents/src/voice/agent.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index 5bde87b95..1c02b8975 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -648,7 +648,6 @@ export class AgentTask extends Agent Date: Wed, 22 Apr 2026 13:58:12 -0700 Subject: [PATCH 5/8] cleanup --- plugins/phonic/src/realtime/realtime_model.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/phonic/src/realtime/realtime_model.ts b/plugins/phonic/src/realtime/realtime_model.ts index abab2859c..d32771715 100644 --- a/plugins/phonic/src/realtime/realtime_model.ts +++ b/plugins/phonic/src/realtime/realtime_model.ts @@ -440,7 +440,6 @@ export class RealtimeSession extends llm.RealtimeSession { if (this.socket) { this.#logger.info('Sending mid-session reset to Phonic'); - this.#logger.debug({ systemPrompt }, 'Phonic reset system prompt'); this.socket.sendReset({ type: 'reset', config: this.buildConfigOptions({ systemPrompt, toolsPayload }), From 88656dcaf6cbb25e2b1e799580c57973650b6611 Mon Sep 17 00:00:00 2001 From: Qiong Zhou Huang Date: Wed, 22 Apr 2026 14:38:48 -0700 Subject: [PATCH 6/8] update --- agents/src/voice/agent.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index 1c02b8975..839796f69 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -529,12 +529,23 @@ export class Agent { }; } +export interface AgentTaskOptions extends AgentOptions { + preserveFunctionCallHistory?: boolean; +} + export class AgentTask extends Agent { private started = false; private future = new Future(); + private _preserveFunctionCallHistory: boolean; #logger = log(); + constructor(options: AgentTaskOptions) { + const { preserveFunctionCallHistory = false, ...rest } = options; + super(rest); + this._preserveFunctionCallHistory = preserveFunctionCallHistory; + } + get done(): boolean { return this.future.done; } @@ -648,6 +659,7 @@ export class AgentTask extends Agent Date: Wed, 22 Apr 2026 14:54:08 -0700 Subject: [PATCH 7/8] update --- agents/src/beta/workflows/task_group.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/agents/src/beta/workflows/task_group.ts b/agents/src/beta/workflows/task_group.ts index 8147c814b..8c96790dd 100644 --- a/agents/src/beta/workflows/task_group.ts +++ b/agents/src/beta/workflows/task_group.ts @@ -37,6 +37,7 @@ export interface TaskGroupOptions { returnExceptions?: boolean; chatCtx?: ChatContext; onTaskCompleted?: (event: TaskCompletedEvent) => Promise; + preserveFunctionCallHistory?: boolean; } export class TaskGroup extends AgentTask { @@ -48,9 +49,15 @@ export class TaskGroup extends AgentTask { private _currentTask?: AgentTask; constructor(options: TaskGroupOptions = {}) { - const { summarizeChatCtx = true, returnExceptions = false, chatCtx, onTaskCompleted } = options; - - super({ instructions: '*empty*', chatCtx }); + const { + summarizeChatCtx = true, + returnExceptions = false, + chatCtx, + onTaskCompleted, + preserveFunctionCallHistory = false, + } = options; + + super({ instructions: '*empty*', chatCtx, preserveFunctionCallHistory }); this._summarizeChatCtx = summarizeChatCtx; this._returnExceptions = returnExceptions; From 994722022a1a2e71d15d524fe8d556befbc91051 Mon Sep 17 00:00:00 2001 From: Qiong Zhou Huang Date: Wed, 22 Apr 2026 16:43:30 -0700 Subject: [PATCH 8/8] add changeset --- .changeset/lemon-stars-float.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/lemon-stars-float.md diff --git a/.changeset/lemon-stars-float.md b/.changeset/lemon-stars-float.md new file mode 100644 index 000000000..7018fbf2c --- /dev/null +++ b/.changeset/lemon-stars-float.md @@ -0,0 +1,6 @@ +--- +'@livekit/agents-plugin-phonic': patch +'@livekit/agents': patch +--- + +add `preserveFunctionCallHistory` option to `AgentTask` and `TaskGroup` and use function call history in Phonic plugin