Skip to content

[stack 2/4] Align main realtime v2 wire and runtime flow #14830

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aibrahim-oai merged 13 commits into from
Mar 17, 2026
Merged

[stack 2/4] Align main realtime v2 wire and runtime flow #14830

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
06e299e
Handle interrupted agent status in tui_app_server
aibrahim-oai Mar 16, 2026
f8c56e6
Align main realtime v2 wire and runtime flow
aibrahim-oai Mar 16, 2026
bb87c8e
Handle realtime audio events in UI clients
aibrahim-oai Mar 16, 2026
3c15083
Branch realtime flow on session kind
aibrahim-oai Mar 16, 2026
6475be2
codex: fix CI failure on PR #14827
aibrahim-oai Mar 16, 2026
55c5505
Remove typed realtime truncate message
aibrahim-oai Mar 16, 2026
03c7c4d
Remove realtime debug logging
aibrahim-oai Mar 16, 2026
062857c
Inline thin realtime runtime helpers
aibrahim-oai Mar 16, 2026
8b3ee3b
Restore preexisting realtime debug logging
aibrahim-oai Mar 16, 2026
81c173c
Strongly type realtime websocket outbound types
aibrahim-oai Mar 16, 2026
0521633
Remove extra multi-agent status allow
aibrahim-oai Mar 16, 2026
b165b0b
Drop stale realtime handoff unit test
aibrahim-oai Mar 16, 2026
9170fe1
Update realtime v2 app-server e2e for response.create
aibrahim-oai Mar 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions codex-rs/app-server-protocol/schema/json/ClientRequest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2771,6 +2771,12 @@
"data": {
"type": "string"
},
"itemId": {
"type": [
"string",
"null"
]
},
"numChannels": {
"format": "uint16",
"minimum": 0.0,
Expand Down
6 changes: 6 additions & 0 deletions codex-rs/app-server-protocol/schema/json/ServerNotification.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2750,6 +2750,12 @@
"data": {
"type": "string"
},
"itemId": {
"type": [
"string",
"null"
]
},
"numChannels": {
"format": "uint16",
"minimum": 0.0,
Expand Down
6 changes: 6 additions & 0 deletions codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12803,6 +12803,12 @@
"data": {
"type": "string"
},
"itemId": {
"type": [
"string",
"null"
]
},
"numChannels": {
"format": "uint16",
"minimum": 0.0,
Expand Down
6 changes: 6 additions & 0 deletions codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10563,6 +10563,12 @@
"data": {
"type": "string"
},
"itemId": {
"type": [
"string",
"null"
]
},
"numChannels": {
"format": "uint16",
"minimum": 0.0,
Expand Down
6 changes: 6 additions & 0 deletions codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeOutputAudioDeltaNotification.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@
"data": {
"type": "string"
},
"itemId": {
"type": [
"string",
"null"
]
},
"numChannels": {
"format": "uint16",
"minimum": 0.0,
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeAudioChunk.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
/**
* EXPERIMENTAL - thread realtime audio chunk.
*/
export type ThreadRealtimeAudioChunk = { data: string, sampleRate: number, numChannels: number, samplesPerChannel: number | null, };
export type ThreadRealtimeAudioChunk = { data: string, sampleRate: number, numChannels: number, samplesPerChannel: number | null, itemId: string | null, };
5 changes: 4 additions & 1 deletion codex-rs/app-server-protocol/src/protocol/common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1577,6 +1577,7 @@ mod tests {
sample_rate: 24_000,
num_channels: 1,
samples_per_channel: Some(512),
item_id: None,
},
},
);
Expand All @@ -1589,7 +1590,8 @@ mod tests {
"data": "AQID",
"sampleRate": 24000,
"numChannels": 1,
"samplesPerChannel": 512
"samplesPerChannel": 512,
"itemId": null
}
}
}),
Expand Down Expand Up @@ -1641,6 +1643,7 @@ mod tests {
sample_rate: 24_000,
num_channels: 1,
samples_per_channel: Some(512),
item_id: None,
},
},
);
Expand Down
5 changes: 5 additions & 0 deletions codex-rs/app-server-protocol/src/protocol/v2.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3652,6 +3652,7 @@ pub struct ThreadRealtimeAudioChunk {
pub sample_rate: u32,
pub num_channels: u16,
pub samples_per_channel: Option<u32>,
pub item_id: Option<String>,
}

impl From<CoreRealtimeAudioFrame> for ThreadRealtimeAudioChunk {
Expand All @@ -3661,12 +3662,14 @@ impl From<CoreRealtimeAudioFrame> for ThreadRealtimeAudioChunk {
sample_rate,
num_channels,
samples_per_channel,
item_id,
} = value;
Self {
data,
sample_rate,
num_channels,
samples_per_channel,
item_id,
}
}
}
Expand All @@ -3678,12 +3681,14 @@ impl From<ThreadRealtimeAudioChunk> for CoreRealtimeAudioFrame {
sample_rate,
num_channels,
samples_per_channel,
item_id,
} = value;
Self {
data,
sample_rate,
num_channels,
samples_per_channel,
item_id,
}
}
}
Expand Down
28 changes: 28 additions & 0 deletions codex-rs/app-server/src/bespoke_event_handling.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,20 @@ pub(crate) async fn apply_bespoke_event_handling(
if let ApiVersion::V2 = api_version {
match event.payload {
RealtimeEvent::SessionUpdated { .. } => {}
RealtimeEvent::InputAudioSpeechStarted(event) => {
let notification = ThreadRealtimeItemAddedNotification {
thread_id: conversation_id.to_string(),
item: serde_json::json!({
"type": "input_audio_buffer.speech_started",
"item_id": event.item_id,
}),
};
outgoing
.send_server_notification(ServerNotification::ThreadRealtimeItemAdded(
notification,
))
.await;
}
RealtimeEvent::InputTranscriptDelta(_) => {}
RealtimeEvent::OutputTranscriptDelta(_) => {}
RealtimeEvent::AudioOut(audio) => {
Expand All @@ -363,6 +377,20 @@ pub(crate) async fn apply_bespoke_event_handling(
)
.await;
}
RealtimeEvent::ResponseCancelled(event) => {
let notification = ThreadRealtimeItemAddedNotification {
thread_id: conversation_id.to_string(),
item: serde_json::json!({
"type": "response.cancelled",
"response_id": event.response_id,
}),
};
outgoing
.send_server_notification(ServerNotification::ThreadRealtimeItemAdded(
notification,
))
.await;
}
RealtimeEvent::ConversationItemAdded(item) => {
let notification = ThreadRealtimeItemAddedNotification {
thread_id: conversation_id.to_string(),
Expand Down
9 changes: 8 additions & 1 deletion codex-rs/app-server/tests/suite/v2/realtime_conversation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
"message": "upstream boom"
}),
],
vec![],
]])
.await;

Expand Down Expand Up @@ -135,6 +136,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
sample_rate: 24_000,
num_channels: 1,
samples_per_channel: Some(480),
item_id: None,
},
})
.await?;
Expand Down Expand Up @@ -191,7 +193,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
let connections = realtime_server.connections();
assert_eq!(connections.len(), 1);
let connection = &connections[0];
assert_eq!(connection.len(), 3);
assert_eq!(connection.len(), 4);
assert_eq!(
connection[0].body_json()["type"].as_str(),
Some("session.update")
Expand All @@ -211,13 +213,18 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
.as_str()
.context("expected websocket request type")?
.to_string(),
connection[3].body_json()["type"]
.as_str()
.context("expected websocket request type")?
.to_string(),
];
request_types.sort();
assert_eq!(
request_types,
[
"conversation.item.create".to_string(),
"input_audio_buffer.append".to_string(),
"response.create".to_string(),
]
);

Expand Down
Loading
Loading