Skip to content

Add realtime output modality and transcript events #17701

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aibrahim-oai merged 18 commits into from
Apr 14, 2026
Merged

Add realtime output modality and transcript events #17701

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
39af7ac
Add realtime connection mode to app-server protocol
aibrahim-oai Apr 13, 2026
d405279
Avoid spelling fixture in realtime transcript test
aibrahim-oai Apr 13, 2026
4d57fd1
Rename realtime start field to output modality
aibrahim-oai Apr 13, 2026
001cdd2
Split realtime transcript notifications
aibrahim-oai Apr 13, 2026
a4320be
codex: address PR review feedback (#17701)
aibrahim-oai Apr 13, 2026
3cd3966
Merge remote-tracking branch 'origin/main' into dev/realtime-connecti…
aibrahim-oai Apr 13, 2026
96ea906
codex: fix CI failure on PR #17701
aibrahim-oai Apr 13, 2026
8374159
codex: address PR review feedback (#17701)
aibrahim-oai Apr 13, 2026
2eb4413
codex: fix CI failure on PR #17701
aibrahim-oai Apr 13, 2026
fa4f84f
codex: fix CI failure on PR #17701
aibrahim-oai Apr 13, 2026
be8ac82
codex: fix CI failure on PR #17701
aibrahim-oai Apr 13, 2026
24fff3c
codex: simplify realtime transcript done handling
aibrahim-oai Apr 14, 2026
b1b26c4
Revert "codex: simplify realtime transcript done handling"
aibrahim-oai Apr 14, 2026
b97dea8
codex: use item done for realtime transcript done
aibrahim-oai Apr 14, 2026
a4a812d
codex: remove realtime transcript correlation state
aibrahim-oai Apr 14, 2026
1000d90
codex: avoid duplicate realtime text done events
aibrahim-oai Apr 14, 2026
52b504e
codex: avoid duplicate realtime audio transcript done
aibrahim-oai Apr 14, 2026
a14eabb
codex: fix realtime transcript done fixture
aibrahim-oai Apr 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions codex-rs/app-server-protocol/schema/json/ClientRequest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1499,6 +1499,13 @@
}
]
},
"RealtimeOutputModality": {
"enum": [
"text",
"audio"
],
"type": "string"
},
"RealtimeVoice": {
"enum": [
"alloy",
Expand Down
52 changes: 47 additions & 5 deletions codex-rs/app-server-protocol/schema/json/ServerNotification.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3384,13 +3384,35 @@
],
"type": "object"
},
"ThreadRealtimeTranscriptUpdatedNotification": {
"ThreadRealtimeTranscriptDeltaNotification": {
"description": "EXPERIMENTAL - flat transcript delta emitted whenever realtime transcript text changes.",
"properties": {
"delta": {
"description": "Live transcript delta from the realtime event.",
"type": "string"
},
"role": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"delta",
"role",
"threadId"
],
"type": "object"
},
"ThreadRealtimeTranscriptDoneNotification": {
"description": "EXPERIMENTAL - final transcript text emitted when realtime completes a transcript part.",
"properties": {
"role": {
"type": "string"
},
"text": {
"description": "Final complete text for the transcript part.",
"type": "string"
},
"threadId": {
Expand Down Expand Up @@ -4949,20 +4971,40 @@
"properties": {
"method": {
"enum": [
"thread/realtime/transcriptUpdated"
"thread/realtime/transcript/delta"
],
"title": "Thread/realtime/transcript/deltaNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/ThreadRealtimeTranscriptDeltaNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/transcript/deltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/realtime/transcript/done"
],
"title": "Thread/realtime/transcriptUpdatedNotificationMethod",
"title": "Thread/realtime/transcript/doneNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/ThreadRealtimeTranscriptUpdatedNotification"
"$ref": "#/definitions/ThreadRealtimeTranscriptDoneNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/transcriptUpdatedNotification",
"title": "Thread/realtime/transcript/doneNotification",
"type": "object"
},
{
Expand Down
63 changes: 57 additions & 6 deletions codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4355,20 +4355,40 @@
"properties": {
"method": {
"enum": [
"thread/realtime/transcriptUpdated"
"thread/realtime/transcript/delta"
],
"title": "Thread/realtime/transcriptUpdatedNotificationMethod",
"title": "Thread/realtime/transcript/deltaNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/v2/ThreadRealtimeTranscriptUpdatedNotification"
"$ref": "#/definitions/v2/ThreadRealtimeTranscriptDeltaNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/transcriptUpdatedNotification",
"title": "Thread/realtime/transcript/deltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/realtime/transcript/done"
],
"title": "Thread/realtime/transcript/doneNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/v2/ThreadRealtimeTranscriptDoneNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/transcript/doneNotification",
"type": "object"
},
{
Expand Down Expand Up @@ -10640,6 +10660,13 @@
],
"type": "string"
},
"RealtimeOutputModality": {
"enum": [
"text",
"audio"
],
"type": "string"
},
"RealtimeVoice": {
"enum": [
"alloy",
Expand Down Expand Up @@ -14010,14 +14037,38 @@
"title": "ThreadRealtimeStartedNotification",
"type": "object"
},
"ThreadRealtimeTranscriptUpdatedNotification": {
"ThreadRealtimeTranscriptDeltaNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - flat transcript delta emitted whenever realtime transcript text changes.",
"properties": {
"delta": {
"description": "Live transcript delta from the realtime event.",
"type": "string"
},
"role": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"delta",
"role",
"threadId"
],
"title": "ThreadRealtimeTranscriptDeltaNotification",
"type": "object"
},
"ThreadRealtimeTranscriptDoneNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - final transcript text emitted when realtime completes a transcript part.",
"properties": {
"role": {
"type": "string"
},
"text": {
"description": "Final complete text for the transcript part.",
"type": "string"
},
"threadId": {
Expand All @@ -14029,7 +14080,7 @@
"text",
"threadId"
],
"title": "ThreadRealtimeTranscriptUpdatedNotification",
"title": "ThreadRealtimeTranscriptDoneNotification",
"type": "object"
},
"ThreadResumeParams": {
Expand Down
63 changes: 57 additions & 6 deletions codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7436,6 +7436,13 @@
],
"type": "string"
},
"RealtimeOutputModality": {
"enum": [
"text",
"audio"
],
"type": "string"
},
"RealtimeVoice": {
"enum": [
"alloy",
Expand Down Expand Up @@ -9605,20 +9612,40 @@
"properties": {
"method": {
"enum": [
"thread/realtime/transcriptUpdated"
"thread/realtime/transcript/delta"
],
"title": "Thread/realtime/transcript/deltaNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/ThreadRealtimeTranscriptDeltaNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/transcript/deltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/realtime/transcript/done"
],
"title": "Thread/realtime/transcriptUpdatedNotificationMethod",
"title": "Thread/realtime/transcript/doneNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/ThreadRealtimeTranscriptUpdatedNotification"
"$ref": "#/definitions/ThreadRealtimeTranscriptDoneNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/transcriptUpdatedNotification",
"title": "Thread/realtime/transcript/doneNotification",
"type": "object"
},
{
Expand Down Expand Up @@ -11858,14 +11885,38 @@
"title": "ThreadRealtimeStartedNotification",
"type": "object"
},
"ThreadRealtimeTranscriptUpdatedNotification": {
"ThreadRealtimeTranscriptDeltaNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - flat transcript delta emitted whenever realtime transcript text changes.",
"properties": {
"delta": {
"description": "Live transcript delta from the realtime event.",
"type": "string"
},
"role": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"delta",
"role",
"threadId"
],
"title": "ThreadRealtimeTranscriptDeltaNotification",
"type": "object"
},
"ThreadRealtimeTranscriptDoneNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - final transcript text emitted when realtime completes a transcript part.",
"properties": {
"role": {
"type": "string"
},
"text": {
"description": "Final complete text for the transcript part.",
"type": "string"
},
"threadId": {
Expand All @@ -11877,7 +11928,7 @@
"text",
"threadId"
],
"title": "ThreadRealtimeTranscriptUpdatedNotification",
"title": "ThreadRealtimeTranscriptDoneNotification",
"type": "object"
},
"ThreadResumeParams": {
Expand Down
9 changes: 5 additions & 4 deletions ...ealtimeTranscriptUpdatedNotification.json → ...dRealtimeTranscriptDeltaNotification.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,22 @@
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - flat transcript delta emitted whenever realtime transcript text changes.",
"properties": {
"role": {
"delta": {
"description": "Live transcript delta from the realtime event.",
"type": "string"
},
"text": {
"role": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"delta",
"role",
"text",
"threadId"
],
"title": "ThreadRealtimeTranscriptUpdatedNotification",
"title": "ThreadRealtimeTranscriptDeltaNotification",
"type": "object"
}
23 changes: 23 additions & 0 deletions codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeTranscriptDoneNotification.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - final transcript text emitted when realtime completes a transcript part.",
"properties": {
"role": {
"type": "string"
},
"text": {
"description": "Final complete text for the transcript part.",
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"role",
"text",
"threadId"
],
"title": "ThreadRealtimeTranscriptDoneNotification",
"type": "object"
}
5 changes: 5 additions & 0 deletions codex-rs/app-server-protocol/schema/typescript/RealtimeOutputModality.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!

// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.

export type RealtimeOutputModality = "text" | "audio";
Loading
Loading