Skip to content

image-gen-core #13290

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
won-openai merged 11 commits into from
Mar 4, 2026
Merged

image-gen-core #13290

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
e916cb4
carGOOOOO
won-openai Mar 4, 2026
c09e9da
input modality, swapped to under-dev and added some integration tests
won-openai Mar 3, 2026
3b988cb
schema changes
won-openai Mar 3, 2026
d95d455
logic changes
won-openai Mar 3, 2026
c993aab
redacted some serdes
won-openai Mar 3, 2026
4c6c93e
added comment
won-openai Mar 3, 2026
288c8cd
comment
won-openai Mar 3, 2026
0a29672
cargo fix
won-openai Mar 4, 2026
5c9a99c
cargo fix
won-openai Mar 4, 2026
58b71c1
schema....
won-openai Mar 4, 2026
bd336b6
schema
won-openai Mar 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions codex-rs/app-server-protocol/schema/json/ClientRequest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1416,6 +1416,40 @@
"title": "WebSearchCallResponseItem",
"type": "object"
},
{
"properties": {
"id": {
"type": "string"
},
"result": {
"type": "string"
},
"revised_prompt": {
"type": [
"string",
"null"
]
},
"status": {
"type": "string"
},
"type": {
"enum": [
"image_generation_call"
],
"title": "ImageGenerationCallResponseItemType",
"type": "string"
}
},
"required": [
"id",
"result",
"status",
"type"
],
"title": "ImageGenerationCallResponseItem",
"type": "object"
},
{
"properties": {
"ghost_commit": {
Expand Down
34 changes: 34 additions & 0 deletions codex-rs/app-server-protocol/schema/json/EventMsg.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4983,6 +4983,40 @@
"title": "WebSearchCallResponseItem",
"type": "object"
},
{
"properties": {
"id": {
"type": "string"
},
"result": {
"type": "string"
},
"revised_prompt": {
"type": [
"string",
"null"
]
},
"status": {
"type": "string"
},
"type": {
"enum": [
"image_generation_call"
],
"title": "ImageGenerationCallResponseItemType",
"type": "string"
}
},
"required": [
"id",
"result",
"status",
"type"
],
"title": "ImageGenerationCallResponseItem",
"type": "object"
},
{
"properties": {
"ghost_commit": {
Expand Down
34 changes: 34 additions & 0 deletions codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11761,6 +11761,40 @@
"title": "WebSearchCallResponseItem",
"type": "object"
},
{
"properties": {
"id": {
"type": "string"
},
"result": {
"type": "string"
},
"revised_prompt": {
"type": [
"string",
"null"
]
},
"status": {
"type": "string"
},
"type": {
"enum": [
"image_generation_call"
],
"title": "ImageGenerationCallResponseItemType",
"type": "string"
}
},
"required": [
"id",
"result",
"status",
"type"
],
"title": "ImageGenerationCallResponseItem",
"type": "object"
},
{
"properties": {
"ghost_commit": {
Expand Down
34 changes: 34 additions & 0 deletions codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9324,6 +9324,40 @@
"title": "WebSearchCallResponseItem",
"type": "object"
},
{
"properties": {
"id": {
"type": "string"
},
"result": {
"type": "string"
},
"revised_prompt": {
"type": [
"string",
"null"
]
},
"status": {
"type": "string"
},
"type": {
"enum": [
"image_generation_call"
],
"title": "ImageGenerationCallResponseItemType",
"type": "string"
}
},
"required": [
"id",
"result",
"status",
"type"
],
"title": "ImageGenerationCallResponseItem",
"type": "object"
},
{
"properties": {
"ghost_commit": {
Expand Down
34 changes: 34 additions & 0 deletions codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -641,6 +641,40 @@
"title": "WebSearchCallResponseItem",
"type": "object"
},
{
"properties": {
"id": {
"type": "string"
},
"result": {
"type": "string"
},
"revised_prompt": {
"type": [
"string",
"null"
]
},
"status": {
"type": "string"
},
"type": {
"enum": [
"image_generation_call"
],
"title": "ImageGenerationCallResponseItemType",
"type": "string"
}
},
"required": [
"id",
"result",
"status",
"type"
],
"title": "ImageGenerationCallResponseItem",
"type": "object"
},
{
"properties": {
"ghost_commit": {
Expand Down
34 changes: 34 additions & 0 deletions codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -691,6 +691,40 @@
"title": "WebSearchCallResponseItem",
"type": "object"
},
{
"properties": {
"id": {
"type": "string"
},
"result": {
"type": "string"
},
"revised_prompt": {
"type": [
"string",
"null"
]
},
"status": {
"type": "string"
},
"type": {
"enum": [
"image_generation_call"
],
"title": "ImageGenerationCallResponseItemType",
"type": "string"
}
},
"required": [
"id",
"result",
"status",
"type"
],
"title": "ImageGenerationCallResponseItem",
"type": "object"
},
{
"properties": {
"ghost_commit": {
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/app-server-protocol/schema/typescript/ResponseItem.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ export type ResponseItem = { "type": "message", role: string, content: Array<Con
/**
* Set when using the Responses API.
*/
call_id: string | null, status: LocalShellStatus, action: LocalShellAction, } | { "type": "function_call", name: string, arguments: string, call_id: string, } | { "type": "function_call_output", call_id: string, output: FunctionCallOutputPayload, } | { "type": "custom_tool_call", status?: string, call_id: string, name: string, input: string, } | { "type": "custom_tool_call_output", call_id: string, output: FunctionCallOutputPayload, } | { "type": "web_search_call", status?: string, action?: WebSearchAction, } | { "type": "ghost_snapshot", ghost_commit: GhostCommit, } | { "type": "compaction", encrypted_content: string, } | { "type": "other" };
call_id: string | null, status: LocalShellStatus, action: LocalShellAction, } | { "type": "function_call", name: string, arguments: string, call_id: string, } | { "type": "function_call_output", call_id: string, output: FunctionCallOutputPayload, } | { "type": "custom_tool_call", status?: string, call_id: string, name: string, input: string, } | { "type": "custom_tool_call_output", call_id: string, output: FunctionCallOutputPayload, } | { "type": "web_search_call", status?: string, action?: WebSearchAction, } | { "type": "image_generation_call", id: string, status: string, revised_prompt?: string, result: string, } | { "type": "ghost_snapshot", ghost_commit: GhostCommit, } | { "type": "compaction", encrypted_content: string, } | { "type": "other" };
6 changes: 6 additions & 0 deletions codex-rs/core/config.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,9 @@
"image_detail_original": {
"type": "boolean"
},
"image_generation": {
"type": "boolean"
},
"include_apply_patch_tool": {
"type": "boolean"
},
Expand Down Expand Up @@ -1752,6 +1755,9 @@
"image_detail_original": {
"type": "boolean"
},
"image_generation": {
"type": "boolean"
},
"include_apply_patch_tool": {
"type": "boolean"
},
Expand Down
1 change: 0 additions & 1 deletion codex-rs/core/src/client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -566,7 +566,6 @@ impl ModelClientSession {
) -> ApiResponsesOptions {
let turn_metadata_header = parse_turn_metadata_header(turn_metadata_header);
let conversation_id = self.client.state.conversation_id.to_string();

ApiResponsesOptions {
conversation_id: Some(conversation_id),
session_source: Some(self.client.state.session_source.clone()),
Expand Down
3 changes: 3 additions & 0 deletions codex-rs/core/src/client_common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,8 @@ pub(crate) mod tools {
Function(ResponsesApiTool),
#[serde(rename = "local_shell")]
LocalShell {},
#[serde(rename = "image_generation")]
ImageGeneration {},
// TODO: Understand why we get an error on web_search although the API docs say it's supported.
// https://platform.openai.com/docs/guides/tools-web-search?api-mode=responses#:~:text=%7B%20type%3A%20%22web_search%22%20%7D%2C
// The `external_web_access` field determines whether the web search is over cached or live content.
Expand All @@ -184,6 +186,7 @@ pub(crate) mod tools {
match self {
ToolSpec::Function(tool) => tool.name.as_str(),
ToolSpec::LocalShell {} => "local_shell",
ToolSpec::ImageGeneration {} => "image_generation",
ToolSpec::WebSearch { .. } => "web_search",
ToolSpec::Freeform(tool) => tool.name.as_str(),
}
Expand Down
1 change: 1 addition & 0 deletions codex-rs/core/src/compact_remote.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ fn should_keep_compacted_history_item(item: &ResponseItem) -> bool {
| ResponseItem::CustomToolCall { .. }
| ResponseItem::CustomToolCallOutput { .. }
| ResponseItem::WebSearchCall { .. }
| ResponseItem::ImageGenerationCall { .. }
| ResponseItem::GhostSnapshot { .. }
| ResponseItem::Other => false,
}
Expand Down
9 changes: 8 additions & 1 deletion codex-rs/core/src/context_manager/history.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,9 @@ impl ContextManager {
// all outputs must have a corresponding function/tool call
normalize::remove_orphan_outputs(&mut self.items);

//rewrite image_gen_calls to messages to support stateless input
normalize::rewrite_image_generation_calls_for_stateless_input(&mut self.items);

// strip images when model does not support them
normalize::strip_images_when_unsupported(input_modalities, &mut self.items);
}
Expand Down Expand Up @@ -374,6 +377,7 @@ impl ContextManager {
| ResponseItem::LocalShellCall { .. }
| ResponseItem::FunctionCall { .. }
| ResponseItem::WebSearchCall { .. }
| ResponseItem::ImageGenerationCall { .. }
| ResponseItem::CustomToolCall { .. }
| ResponseItem::Compaction { .. }
| ResponseItem::GhostSnapshot { .. }
Expand Down Expand Up @@ -402,7 +406,8 @@ fn truncate_function_output_payload(
}

/// API messages include every non-system item (user/assistant messages, reasoning,
/// tool calls, tool outputs, shell calls, and web-search calls).
/// tool calls, tool outputs, shell calls, web-search calls, and image-generation
/// calls).
fn is_api_message(message: &ResponseItem) -> bool {
match message {
ResponseItem::Message { role, .. } => role.as_str() != "system",
Expand All @@ -413,6 +418,7 @@ fn is_api_message(message: &ResponseItem) -> bool {
| ResponseItem::LocalShellCall { .. }
| ResponseItem::Reasoning { .. }
| ResponseItem::WebSearchCall { .. }
| ResponseItem::ImageGenerationCall { .. }
| ResponseItem::Compaction { .. } => true,
ResponseItem::GhostSnapshot { .. } => false,
ResponseItem::Other => false,
Expand Down Expand Up @@ -600,6 +606,7 @@ fn is_model_generated_item(item: &ResponseItem) -> bool {
ResponseItem::Reasoning { .. }
| ResponseItem::FunctionCall { .. }
| ResponseItem::WebSearchCall { .. }
| ResponseItem::ImageGenerationCall { .. }
| ResponseItem::CustomToolCall { .. }
| ResponseItem::LocalShellCall { .. }
| ResponseItem::Compaction { .. } => true,
Expand Down
Loading
Loading