From 94b0e35f436aa17b304fc228ec9adcbae1f22474 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 14:34:45 -0700 Subject: [PATCH 01/14] Move string truncation helpers into codex-utils-string Co-authored-by: Codex --- codex-rs/Cargo.lock | 11 + codex-rs/Cargo.toml | 2 + codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/codex.rs | 4 +- .../src/codex/rollout_reconstruction_tests.rs | 16 +- codex-rs/core/src/codex_tests.rs | 2 +- codex-rs/core/src/compact.rs | 6 +- codex-rs/core/src/context_manager/history.rs | 12 +- .../core/src/context_manager/history_tests.rs | 6 +- codex-rs/core/src/error.rs | 4 +- codex-rs/core/src/guardian/prompt.rs | 6 +- codex-rs/core/src/lib.rs | 1 - codex-rs/core/src/memories/prompts.rs | 4 +- .../core/src/models_manager/model_info.rs | 2 +- codex-rs/core/src/realtime_context.rs | 4 +- codex-rs/core/src/rollout/list.rs | 2 +- codex-rs/core/src/rollout/recorder.rs | 8 +- codex-rs/core/src/rollout/recorder_tests.rs | 2 +- codex-rs/core/src/rollout/tests.rs | 2 +- codex-rs/core/src/state/session.rs | 2 +- codex-rs/core/src/tools/code_mode/mod.rs | 6 +- codex-rs/core/src/tools/context.rs | 4 +- codex-rs/core/src/tools/js_repl/mod.rs | 4 +- codex-rs/core/src/tools/mod.rs | 6 +- codex-rs/core/src/truncate.rs | 363 ------------------ codex-rs/core/src/unified_exec/process.rs | 4 +- .../core/src/unified_exec/process_manager.rs | 2 +- codex-rs/protocol/Cargo.toml | 1 + codex-rs/protocol/src/protocol.rs | 46 +++ codex-rs/utils/output-truncation/BUILD.bazel | 6 + codex-rs/utils/output-truncation/Cargo.toml | 15 + codex-rs/utils/output-truncation/src/lib.rs | 142 +++++++ .../output-truncation/src/tests.rs} | 106 ++--- codex-rs/utils/string/src/lib.rs | 9 + codex-rs/utils/string/src/truncate.rs | 156 ++++++++ codex-rs/utils/string/src/truncate/tests.rs | 71 ++++ 36 files changed, 551 insertions(+), 487 deletions(-) delete mode 100644 codex-rs/core/src/truncate.rs create mode 100644 codex-rs/utils/output-truncation/BUILD.bazel create mode 100644 codex-rs/utils/output-truncation/Cargo.toml create mode 100644 codex-rs/utils/output-truncation/src/lib.rs rename codex-rs/{core/src/truncate_tests.rs => utils/output-truncation/src/tests.rs} (77%) create mode 100644 codex-rs/utils/string/src/truncate.rs create mode 100644 codex-rs/utils/string/src/truncate/tests.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 935fc413a58..697b8f5d321 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1902,6 +1902,7 @@ dependencies = [ "codex-utils-cargo-bin", "codex-utils-home-dir", "codex-utils-image", + "codex-utils-output-truncation", "codex-utils-pty", "codex-utils-readiness", "codex-utils-stream-parser", @@ -2399,6 +2400,7 @@ dependencies = [ "codex-git-utils", "codex-utils-absolute-path", "codex-utils-image", + "codex-utils-string", "icu_decimal", "icu_locale_core", "icu_provider", @@ -2891,6 +2893,15 @@ dependencies = [ "codex-ollama", ] +[[package]] +name = "codex-utils-output-truncation" +version = "0.0.0" +dependencies = [ + "codex-protocol", + "codex-utils-string", + "pretty_assertions", +] + [[package]] name = "codex-utils-pty" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index e071ec807e7..ebd88c945f2 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -65,6 +65,7 @@ members = [ "utils/sleep-inhibitor", "utils/approval-presets", "utils/oss", + "utils/output-truncation", "utils/fuzzy-match", "utils/stream-parser", "codex-client", @@ -154,6 +155,7 @@ codex-utils-home-dir = { path = "utils/home-dir" } codex-utils-image = { path = "utils/image" } codex-utils-json-to-toml = { path = "utils/json-to-toml" } codex-utils-oss = { path = "utils/oss" } +codex-utils-output-truncation = { path = "utils/output-truncation" } codex-utils-pty = { path = "utils/pty" } codex-utils-readiness = { path = "utils/readiness" } codex-utils-rustls-provider = { path = "utils/rustls-provider" } diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 82ea2600558..51dd305b35a 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -55,6 +55,7 @@ codex-utils-absolute-path = { workspace = true } codex-utils-cache = { workspace = true } codex-utils-image = { workspace = true } codex-utils-home-dir = { workspace = true } +codex-utils-output-truncation = { workspace = true } codex-utils-pty = { workspace = true } codex-utils-readiness = { workspace = true } codex-secrets = { workspace = true } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index da5f6e0d5ac..ccef26ef7b9 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -46,7 +46,6 @@ use crate::stream_events_utils::handle_output_item_done; use crate::stream_events_utils::last_assistant_message_from_item; use crate::stream_events_utils::raw_assistant_output_text_from_item; use crate::stream_events_utils::record_completed_response_item; -use crate::truncate::TruncationPolicy; use crate::turn_metadata::TurnMetadataState; use crate::util::error_or_panic; use async_channel::Receiver; @@ -117,6 +116,7 @@ use codex_protocol::request_user_input::RequestUserInputResponse; use codex_rmcp_client::ElicitationResponse; use codex_rmcp_client::OAuthCredentialsStoreMode; use codex_terminal_detection::user_agent; +use codex_utils_output_truncation::TruncationPolicy; use codex_utils_stream_parser::AssistantTextChunk; use codex_utils_stream_parser::AssistantTextStreamParser; use codex_utils_stream_parser::ProposedPlanSegment; @@ -1007,7 +1007,7 @@ impl TurnContext { user_instructions: self.user_instructions.clone(), developer_instructions: self.developer_instructions.clone(), final_output_json_schema: self.final_output_json_schema.clone(), - truncation_policy: Some(self.truncation_policy.into()), + truncation_policy: Some(self.truncation_policy), } } diff --git a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs index 906c72c92ac..1ed668d56fd 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs @@ -77,7 +77,7 @@ async fn record_initial_history_resumed_bare_turn_context_does_not_hydrate_previ user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), + truncation_policy: Some(turn_context.truncation_policy), }; let rollout_items = vec![RolloutItem::TurnContext(previous_context_item)]; @@ -116,7 +116,7 @@ async fn record_initial_history_resumed_hydrates_previous_turn_settings_from_lif user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), + truncation_policy: Some(turn_context.truncation_policy), }; let turn_id = previous_context_item .turn_id @@ -866,7 +866,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), + truncation_policy: Some(turn_context.truncation_policy), }; let previous_turn_id = previous_context_item .turn_id @@ -938,7 +938,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), + truncation_policy: Some(turn_context.truncation_policy), })) .expect("serialize expected reference context item") ); @@ -967,7 +967,7 @@ async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_tu user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), + truncation_policy: Some(turn_context.truncation_policy), }; let previous_turn_id = previous_context_item .turn_id @@ -1073,7 +1073,7 @@ async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_fo user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), + truncation_policy: Some(turn_context.truncation_policy), }; let rollout_items = vec![ @@ -1175,7 +1175,7 @@ async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clea user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), + truncation_policy: Some(turn_context.truncation_policy), }; let previous_turn_id = previous_context_item .turn_id @@ -1319,7 +1319,7 @@ async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clear user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), + truncation_policy: Some(turn_context.truncation_policy), }; let previous_turn_id = previous_context_item .turn_id diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 8846f0a643d..0ba0a1beb0c 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -1287,7 +1287,7 @@ async fn record_initial_history_forked_hydrates_previous_turn_settings() { user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy.into()), + truncation_policy: Some(turn_context.truncation_policy), }; let turn_id = previous_context_item .turn_id diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 7686c5b65cd..82fd5c15e30 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -15,9 +15,6 @@ use crate::protocol::CompactedItem; use crate::protocol::EventMsg; use crate::protocol::TurnStartedEvent; use crate::protocol::WarningEvent; -use crate::truncate::TruncationPolicy; -use crate::truncate::approx_token_count; -use crate::truncate::truncate_text; use crate::util::backoff; use codex_protocol::items::ContextCompactionItem; use codex_protocol::items::TurnItem; @@ -25,6 +22,9 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::user_input::UserInput; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::approx_token_count; +use codex_utils_output_truncation::truncate_text; use futures::prelude::*; use tracing::error; diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index b518d8ce4ff..ec2df30d3bd 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -3,12 +3,6 @@ use crate::context_manager::normalize; use crate::event_mapping::has_non_contextual_dev_message_content; use crate::event_mapping::is_contextual_dev_message_content; use crate::event_mapping::is_contextual_user_message_content; -use crate::truncate::TruncationPolicy; -use crate::truncate::approx_bytes_for_tokens; -use crate::truncate::approx_token_count; -use crate::truncate::approx_tokens_from_byte_count_i64; -use crate::truncate::truncate_function_output_items_with_policy; -use crate::truncate::truncate_text; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_protocol::models::BaseInstructions; @@ -25,6 +19,12 @@ use codex_protocol::protocol::TokenUsageInfo; use codex_protocol::protocol::TurnContextItem; use codex_utils_cache::BlockingLruCache; use codex_utils_cache::sha1_digest; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::approx_bytes_for_tokens; +use codex_utils_output_truncation::approx_token_count; +use codex_utils_output_truncation::approx_tokens_from_byte_count_i64; +use codex_utils_output_truncation::truncate_function_output_items_with_policy; +use codex_utils_output_truncation::truncate_text; use std::num::NonZeroUsize; use std::ops::Deref; use std::sync::LazyLock; diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 0c92edcfb62..3c508e05ff7 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -1,6 +1,4 @@ use super::*; -use crate::truncate; -use crate::truncate::TruncationPolicy; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_git_utils::GhostCommit; @@ -23,6 +21,8 @@ use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::TurnContextItem; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::truncate_text; use image::ImageBuffer; use image::ImageFormat; use image::Rgba; @@ -179,7 +179,7 @@ fn reasoning_with_encrypted_content(len: usize) -> ResponseItem { } fn truncate_exec_output(content: &str) -> String { - truncate::truncate_text(content, TruncationPolicy::Tokens(EXEC_FORMAT_MAX_TOKENS)) + truncate_text(content, TruncationPolicy::Tokens(EXEC_FORMAT_MAX_TOKENS)) } fn approx_token_count_for_text(text: &str) -> i64 { diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index b84180d1607..b0c55a55c64 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -2,8 +2,6 @@ use crate::exec::ExecToolCallOutput; use crate::network_policy_decision::NetworkPolicyDecisionPayload; use crate::token_data::KnownPlan; use crate::token_data::PlanType; -use crate::truncate::TruncationPolicy; -use crate::truncate::truncate_text; use chrono::DateTime; use chrono::Datelike; use chrono::Local; @@ -15,6 +13,8 @@ use codex_protocol::ThreadId; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::RateLimitSnapshot; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::truncate_text; use reqwest::StatusCode; use serde_json; use std::io; diff --git a/codex-rs/core/src/guardian/prompt.rs b/codex-rs/core/src/guardian/prompt.rs index be029164877..5741a65fc06 100644 --- a/codex-rs/core/src/guardian/prompt.rs +++ b/codex-rs/core/src/guardian/prompt.rs @@ -7,9 +7,9 @@ use serde_json::Value; use crate::codex::Session; use crate::compact::content_items_to_text; use crate::event_mapping::is_contextual_user_message_content; -use crate::truncate::approx_bytes_for_tokens; -use crate::truncate::approx_token_count; -use crate::truncate::approx_tokens_from_byte_count; +use codex_utils_output_truncation::approx_bytes_for_tokens; +use codex_utils_output_truncation::approx_token_count; +use codex_utils_output_truncation::approx_tokens_from_byte_count; use super::GUARDIAN_MAX_MESSAGE_ENTRY_TOKENS; use super::GUARDIAN_MAX_MESSAGE_TRANSCRIPT_TOKENS; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index decebc84e07..d4a8ae93aa0 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -78,7 +78,6 @@ mod stream_events_utils; pub mod test_support; mod text_encoding; pub use codex_login::token_data; -mod truncate; mod unified_exec; pub mod windows_sandbox; pub use client::X_RESPONSESAPI_INCLUDE_TIMING_METRICS_HEADER; diff --git a/codex-rs/core/src/memories/prompts.rs b/codex-rs/core/src/memories/prompts.rs index 1659e1c1c77..851179426f0 100644 --- a/codex-rs/core/src/memories/prompts.rs +++ b/codex-rs/core/src/memories/prompts.rs @@ -1,13 +1,13 @@ use crate::memories::memory_root; use crate::memories::phase_one; use crate::memories::storage::rollout_summary_file_stem_from_parts; -use crate::truncate::TruncationPolicy; -use crate::truncate::truncate_text; use askama::Template; use codex_protocol::openai_models::ModelInfo; use codex_state::Phase2InputSelection; use codex_state::Stage1Output; use codex_state::Stage1OutputRef; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::truncate_text; use std::path::Path; use tokio::fs; use tracing::warn; diff --git a/codex-rs/core/src/models_manager/model_info.rs b/codex-rs/core/src/models_manager/model_info.rs index 7d3c6e9d10a..055a6459d46 100644 --- a/codex-rs/core/src/models_manager/model_info.rs +++ b/codex-rs/core/src/models_manager/model_info.rs @@ -10,8 +10,8 @@ use codex_protocol::openai_models::WebSearchToolType; use codex_protocol::openai_models::default_input_modalities; use crate::config::Config; -use crate::truncate::approx_bytes_for_tokens; use codex_features::Feature; +use codex_utils_output_truncation::approx_bytes_for_tokens; use tracing::warn; pub const BASE_INSTRUCTIONS: &str = include_str!("../../prompt.md"); diff --git a/codex-rs/core/src/realtime_context.rs b/codex-rs/core/src/realtime_context.rs index bcb24c56739..1b845db1e9d 100644 --- a/codex-rs/core/src/realtime_context.rs +++ b/codex-rs/core/src/realtime_context.rs @@ -1,13 +1,13 @@ use crate::codex::Session; use crate::compact::content_items_to_text; use crate::event_mapping::is_contextual_user_message_content; -use crate::truncate::TruncationPolicy; -use crate::truncate::truncate_text; use chrono::Utc; use codex_git_utils::resolve_root_git_project_for_trust; use codex_protocol::models::ResponseItem; use codex_state::SortKey; use codex_state::ThreadMetadata; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::truncate_text; use dirs::home_dir; use std::cmp::Reverse; use std::collections::HashMap; diff --git a/codex-rs/core/src/rollout/list.rs b/codex-rs/core/src/rollout/list.rs index bf32e23c8d3..6ae5908d8aa 100644 --- a/codex-rs/core/src/rollout/list.rs +++ b/codex-rs/core/src/rollout/list.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use std::cmp::Reverse; use std::ffi::OsStr; -use std::io::{self}; +use std::io; use std::num::NonZero; use std::ops::ControlFlow; use std::path::Path; diff --git a/codex-rs/core/src/rollout/recorder.rs b/codex-rs/core/src/rollout/recorder.rs index 25e7ab20343..30b0734df15 100644 --- a/codex-rs/core/src/rollout/recorder.rs +++ b/codex-rs/core/src/rollout/recorder.rs @@ -1,7 +1,7 @@ //! Persist Codex session rollouts (.jsonl) so sessions can be replayed or inspected later. +use std::fs; use std::fs::File; -use std::fs::{self}; use std::io::Error as IoError; use std::path::Path; use std::path::PathBuf; @@ -17,8 +17,8 @@ use time::format_description::FormatItem; use time::format_description::well_known::Rfc3339; use time::macros::format_description; use tokio::io::AsyncWriteExt; +use tokio::sync::mpsc; use tokio::sync::mpsc::Sender; -use tokio::sync::mpsc::{self}; use tokio::sync::oneshot; use tracing::info; use tracing::trace; @@ -44,8 +44,6 @@ use crate::default_client::originator; use crate::path_utils; use crate::state_db; use crate::state_db::StateDbHandle; -use crate::truncate::TruncationPolicy; -use crate::truncate::truncate_text; use codex_git_utils::collect_git_info; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::GitInfo as ProtocolGitInfo; @@ -58,6 +56,8 @@ use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_state::StateRuntime; use codex_state::ThreadMetadataBuilder; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::truncate_text; /// Records all [`ResponseItem`]s for a session and flushes them to disk after /// every update. diff --git a/codex-rs/core/src/rollout/recorder_tests.rs b/codex-rs/core/src/rollout/recorder_tests.rs index 8ca7b58a6b5..9443c877b4b 100644 --- a/codex-rs/core/src/rollout/recorder_tests.rs +++ b/codex-rs/core/src/rollout/recorder_tests.rs @@ -10,8 +10,8 @@ use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::UserMessageEvent; use pretty_assertions::assert_eq; +use std::fs; use std::fs::File; -use std::fs::{self}; use std::io::Write; use std::path::Path; use std::path::PathBuf; diff --git a/codex-rs/core/src/rollout/tests.rs b/codex-rs/core/src/rollout/tests.rs index 44e536e50ef..17c1792d32e 100644 --- a/codex-rs/core/src/rollout/tests.rs +++ b/codex-rs/core/src/rollout/tests.rs @@ -1,9 +1,9 @@ #![allow(clippy::unwrap_used, clippy::expect_used)] use std::ffi::OsStr; +use std::fs; use std::fs::File; use std::fs::FileTimes; -use std::fs::{self}; use std::io::Write; use std::path::Path; diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 81978160a62..1a14236163d 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -13,8 +13,8 @@ use crate::protocol::RateLimitSnapshot; use crate::protocol::TokenUsage; use crate::protocol::TokenUsageInfo; use crate::session_startup_prewarm::SessionStartupPrewarmHandle; -use crate::truncate::TruncationPolicy; use codex_protocol::protocol::TurnContextItem; +use codex_utils_output_truncation::TruncationPolicy; /// Persistent, session-scoped state previously stored directly on `Session`. pub(crate) struct SessionState { diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index a4838d24634..a1f21a62276 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -27,11 +27,11 @@ use crate::tools::parallel::ToolCallRuntime; use crate::tools::router::ToolCall; use crate::tools::router::ToolCallSource; use crate::tools::router::ToolRouterParams; -use crate::truncate::TruncationPolicy; -use crate::truncate::formatted_truncate_text_content_items_with_policy; -use crate::truncate::truncate_function_output_items_with_policy; use crate::unified_exec::resolve_max_tokens; use codex_features::Feature; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::formatted_truncate_text_content_items_with_policy; +use codex_utils_output_truncation::truncate_function_output_items_with_policy; pub(crate) use execute_handler::CodeModeExecuteHandler; use response_adapter::into_function_call_output_content_items; diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs index 74efb0989ba..12db451027e 100644 --- a/codex-rs/core/src/tools/context.rs +++ b/codex-rs/core/src/tools/context.rs @@ -4,8 +4,6 @@ use crate::codex::TurnContext; use crate::tools::TELEMETRY_PREVIEW_MAX_BYTES; use crate::tools::TELEMETRY_PREVIEW_MAX_LINES; use crate::tools::TELEMETRY_PREVIEW_TRUNCATION_NOTICE; -use crate::truncate::TruncationPolicy; -use crate::truncate::formatted_truncate_text; use crate::turn_diff_tracker::TurnDiffTracker; use crate::unified_exec::resolve_max_tokens; use codex_protocol::mcp::CallToolResult; @@ -16,6 +14,8 @@ use codex_protocol::models::ResponseInputItem; use codex_protocol::models::SearchToolCallParams; use codex_protocol::models::ShellToolCallParams; use codex_protocol::models::function_call_output_content_items_to_text; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::formatted_truncate_text; use codex_utils_string::take_bytes_at_char_boundary; use serde::Serialize; use serde_json::Value as JsonValue; diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index 73c0b34ead6..66a09c7e8f2 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -43,12 +43,12 @@ use crate::sandboxing::ExecOptions; use crate::sandboxing::SandboxPermissions; use crate::tools::ToolRouter; use crate::tools::context::SharedTurnDiffTracker; -use crate::truncate::TruncationPolicy; -use crate::truncate::truncate_text; use codex_sandboxing::SandboxCommand; use codex_sandboxing::SandboxManager; use codex_sandboxing::SandboxTransformRequest; use codex_sandboxing::SandboxablePreference; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::truncate_text; pub(crate) const JS_REPL_PRAGMA_PREFIX: &str = "// codex-js-repl:"; const KERNEL_SOURCE: &str = include_str!("kernel.js"); diff --git a/codex-rs/core/src/tools/mod.rs b/codex-rs/core/src/tools/mod.rs index 4e495190ec9..87a01c8e3b4 100644 --- a/codex-rs/core/src/tools/mod.rs +++ b/codex-rs/core/src/tools/mod.rs @@ -15,9 +15,9 @@ pub mod sandboxing; pub mod spec; use crate::exec::ExecToolCallOutput; -use crate::truncate::TruncationPolicy; -use crate::truncate::formatted_truncate_text; -use crate::truncate::truncate_text; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::formatted_truncate_text; +use codex_utils_output_truncation::truncate_text; pub use router::ToolRouter; use serde::Serialize; diff --git a/codex-rs/core/src/truncate.rs b/codex-rs/core/src/truncate.rs deleted file mode 100644 index 707fbe22ef4..00000000000 --- a/codex-rs/core/src/truncate.rs +++ /dev/null @@ -1,363 +0,0 @@ -//! Utilities for truncating large chunks of output while preserving a prefix -//! and suffix on UTF-8 boundaries, and helpers for line/token‑based truncation -//! used across the core crate. - -use codex_protocol::models::FunctionCallOutputContentItem; -use codex_protocol::openai_models::TruncationMode; -use codex_protocol::openai_models::TruncationPolicyConfig; -use codex_protocol::protocol::TruncationPolicy as ProtocolTruncationPolicy; - -const APPROX_BYTES_PER_TOKEN: usize = 4; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum TruncationPolicy { - Bytes(usize), - Tokens(usize), -} - -impl From for ProtocolTruncationPolicy { - fn from(value: TruncationPolicy) -> Self { - match value { - TruncationPolicy::Bytes(bytes) => Self::Bytes(bytes), - TruncationPolicy::Tokens(tokens) => Self::Tokens(tokens), - } - } -} - -impl From for TruncationPolicy { - fn from(config: TruncationPolicyConfig) -> Self { - match config.mode { - TruncationMode::Bytes => Self::Bytes(config.limit as usize), - TruncationMode::Tokens => Self::Tokens(config.limit as usize), - } - } -} - -impl TruncationPolicy { - /// Returns a token budget derived from this policy. - /// - /// - For `Tokens`, this is the explicit token limit. - /// - For `Bytes`, this is an approximate token budget using the global - /// bytes-per-token heuristic. - pub fn token_budget(&self) -> usize { - match self { - TruncationPolicy::Bytes(bytes) => { - usize::try_from(approx_tokens_from_byte_count(*bytes)).unwrap_or(usize::MAX) - } - TruncationPolicy::Tokens(tokens) => *tokens, - } - } - - /// Returns a byte budget derived from this policy. - /// - /// - For `Bytes`, this is the explicit byte limit. - /// - For `Tokens`, this is an approximate byte budget using the global - /// bytes-per-token heuristic. - pub fn byte_budget(&self) -> usize { - match self { - TruncationPolicy::Bytes(bytes) => *bytes, - TruncationPolicy::Tokens(tokens) => approx_bytes_for_tokens(*tokens), - } - } -} - -impl std::ops::Mul for TruncationPolicy { - type Output = Self; - - fn mul(self, multiplier: f64) -> Self::Output { - match self { - TruncationPolicy::Bytes(bytes) => { - TruncationPolicy::Bytes((bytes as f64 * multiplier).ceil() as usize) - } - TruncationPolicy::Tokens(tokens) => { - TruncationPolicy::Tokens((tokens as f64 * multiplier).ceil() as usize) - } - } - } -} - -pub(crate) fn formatted_truncate_text(content: &str, policy: TruncationPolicy) -> String { - if content.len() <= policy.byte_budget() { - return content.to_string(); - } - let total_lines = content.lines().count(); - let result = truncate_text(content, policy); - format!("Total output lines: {total_lines}\n\n{result}") -} - -pub(crate) fn truncate_text(content: &str, policy: TruncationPolicy) -> String { - match policy { - TruncationPolicy::Bytes(_) => truncate_with_byte_estimate(content, policy), - TruncationPolicy::Tokens(_) => { - let (truncated, _) = truncate_with_token_budget(content, policy); - truncated - } - } -} - -pub(crate) fn formatted_truncate_text_content_items_with_policy( - items: &[FunctionCallOutputContentItem], - policy: TruncationPolicy, -) -> (Vec, Option) { - let text_segments = items - .iter() - .filter_map(|item| match item { - FunctionCallOutputContentItem::InputText { text } => Some(text.as_str()), - FunctionCallOutputContentItem::InputImage { .. } => None, - }) - .collect::>(); - - if text_segments.is_empty() { - return (items.to_vec(), None); - } - - let mut combined = String::new(); - for text in &text_segments { - if !combined.is_empty() { - combined.push('\n'); - } - combined.push_str(text); - } - - if combined.len() <= policy.byte_budget() { - return (items.to_vec(), None); - } - - let mut out = vec![FunctionCallOutputContentItem::InputText { - text: formatted_truncate_text(&combined, policy), - }]; - out.extend(items.iter().filter_map(|item| match item { - FunctionCallOutputContentItem::InputImage { image_url, detail } => { - Some(FunctionCallOutputContentItem::InputImage { - image_url: image_url.clone(), - detail: *detail, - }) - } - FunctionCallOutputContentItem::InputText { .. } => None, - })); - - (out, Some(approx_token_count(&combined))) -} - -/// Globally truncate function output items to fit within the given -/// truncation policy's budget, preserving as many text/image items as -/// possible and appending a summary for any omitted text items. -pub(crate) fn truncate_function_output_items_with_policy( - items: &[FunctionCallOutputContentItem], - policy: TruncationPolicy, -) -> Vec { - let mut out: Vec = Vec::with_capacity(items.len()); - let mut remaining_budget = match policy { - TruncationPolicy::Bytes(_) => policy.byte_budget(), - TruncationPolicy::Tokens(_) => policy.token_budget(), - }; - let mut omitted_text_items = 0usize; - - for it in items { - match it { - FunctionCallOutputContentItem::InputText { text } => { - if remaining_budget == 0 { - omitted_text_items += 1; - continue; - } - - let cost = match policy { - TruncationPolicy::Bytes(_) => text.len(), - TruncationPolicy::Tokens(_) => approx_token_count(text), - }; - - if cost <= remaining_budget { - out.push(FunctionCallOutputContentItem::InputText { text: text.clone() }); - remaining_budget = remaining_budget.saturating_sub(cost); - } else { - let snippet_policy = match policy { - TruncationPolicy::Bytes(_) => TruncationPolicy::Bytes(remaining_budget), - TruncationPolicy::Tokens(_) => TruncationPolicy::Tokens(remaining_budget), - }; - let snippet = truncate_text(text, snippet_policy); - if snippet.is_empty() { - omitted_text_items += 1; - } else { - out.push(FunctionCallOutputContentItem::InputText { text: snippet }); - } - remaining_budget = 0; - } - } - FunctionCallOutputContentItem::InputImage { image_url, detail } => { - out.push(FunctionCallOutputContentItem::InputImage { - image_url: image_url.clone(), - detail: *detail, - }); - } - } - } - - if omitted_text_items > 0 { - out.push(FunctionCallOutputContentItem::InputText { - text: format!("[omitted {omitted_text_items} text items ...]"), - }); - } - - out -} - -/// Truncate the middle of a UTF-8 string to at most `max_tokens` tokens, -/// preserving the beginning and the end. Returns the possibly truncated string -/// and `Some(original_token_count)` if truncation occurred; otherwise returns -/// the original string and `None`. -fn truncate_with_token_budget(s: &str, policy: TruncationPolicy) -> (String, Option) { - if s.is_empty() { - return (String::new(), None); - } - let max_tokens = policy.token_budget(); - - let byte_len = s.len(); - if max_tokens > 0 && byte_len <= approx_bytes_for_tokens(max_tokens) { - return (s.to_string(), None); - } - - let truncated = truncate_with_byte_estimate(s, policy); - let approx_total_usize = approx_token_count(s); - let approx_total = u64::try_from(approx_total_usize).unwrap_or(u64::MAX); - if truncated == s { - (truncated, None) - } else { - (truncated, Some(approx_total)) - } -} - -/// Truncate a string using a byte budget derived from the token budget, without -/// performing any real tokenization. This keeps the logic purely byte-based and -/// uses a bytes placeholder in the truncated output. -fn truncate_with_byte_estimate(s: &str, policy: TruncationPolicy) -> String { - if s.is_empty() { - return String::new(); - } - - let total_chars = s.chars().count(); - let max_bytes = policy.byte_budget(); - - if max_bytes == 0 { - // No budget to show content; just report that everything was truncated. - let marker = format_truncation_marker( - policy, - removed_units_for_source(policy, s.len(), total_chars), - ); - return marker; - } - - if s.len() <= max_bytes { - return s.to_string(); - } - - let total_bytes = s.len(); - - let (left_budget, right_budget) = split_budget(max_bytes); - - let (removed_chars, left, right) = split_string(s, left_budget, right_budget); - - let marker = format_truncation_marker( - policy, - removed_units_for_source(policy, total_bytes.saturating_sub(max_bytes), removed_chars), - ); - - assemble_truncated_output(left, right, &marker) -} - -fn split_string(s: &str, beginning_bytes: usize, end_bytes: usize) -> (usize, &str, &str) { - if s.is_empty() { - return (0, "", ""); - } - - let len = s.len(); - let tail_start_target = len.saturating_sub(end_bytes); - let mut prefix_end = 0usize; - let mut suffix_start = len; - let mut removed_chars = 0usize; - let mut suffix_started = false; - - for (idx, ch) in s.char_indices() { - let char_end = idx + ch.len_utf8(); - if char_end <= beginning_bytes { - prefix_end = char_end; - continue; - } - - if idx >= tail_start_target { - if !suffix_started { - suffix_start = idx; - suffix_started = true; - } - continue; - } - - removed_chars = removed_chars.saturating_add(1); - } - - if suffix_start < prefix_end { - suffix_start = prefix_end; - } - - let before = &s[..prefix_end]; - let after = &s[suffix_start..]; - - (removed_chars, before, after) -} - -fn format_truncation_marker(policy: TruncationPolicy, removed_count: u64) -> String { - match policy { - TruncationPolicy::Tokens(_) => format!("…{removed_count} tokens truncated…"), - TruncationPolicy::Bytes(_) => format!("…{removed_count} chars truncated…"), - } -} - -fn split_budget(budget: usize) -> (usize, usize) { - let left = budget / 2; - (left, budget - left) -} - -fn removed_units_for_source( - policy: TruncationPolicy, - removed_bytes: usize, - removed_chars: usize, -) -> u64 { - match policy { - TruncationPolicy::Tokens(_) => approx_tokens_from_byte_count(removed_bytes), - TruncationPolicy::Bytes(_) => u64::try_from(removed_chars).unwrap_or(u64::MAX), - } -} - -fn assemble_truncated_output(prefix: &str, suffix: &str, marker: &str) -> String { - let mut out = String::with_capacity(prefix.len() + marker.len() + suffix.len() + 1); - out.push_str(prefix); - out.push_str(marker); - out.push_str(suffix); - out -} - -pub(crate) fn approx_token_count(text: &str) -> usize { - let len = text.len(); - len.saturating_add(APPROX_BYTES_PER_TOKEN.saturating_sub(1)) / APPROX_BYTES_PER_TOKEN -} - -pub(crate) fn approx_bytes_for_tokens(tokens: usize) -> usize { - tokens.saturating_mul(APPROX_BYTES_PER_TOKEN) -} - -pub(crate) fn approx_tokens_from_byte_count(bytes: usize) -> u64 { - let bytes_u64 = bytes as u64; - bytes_u64.saturating_add((APPROX_BYTES_PER_TOKEN as u64).saturating_sub(1)) - / (APPROX_BYTES_PER_TOKEN as u64) -} - -pub(crate) fn approx_tokens_from_byte_count_i64(bytes: i64) -> i64 { - if bytes <= 0 { - return 0; - } - let bytes = usize::try_from(bytes).unwrap_or(usize::MAX); - i64::try_from(approx_tokens_from_byte_count(bytes)).unwrap_or(i64::MAX) -} - -#[cfg(test)] -#[path = "truncate_tests.rs"] -mod tests; diff --git a/codex-rs/core/src/unified_exec/process.rs b/codex-rs/core/src/unified_exec/process.rs index 58faed27de9..68cf18bce11 100644 --- a/codex-rs/core/src/unified_exec/process.rs +++ b/codex-rs/core/src/unified_exec/process.rs @@ -15,9 +15,9 @@ use tokio_util::sync::CancellationToken; use crate::exec::ExecToolCallOutput; use crate::exec::StreamOutput; use crate::exec::is_likely_sandbox_denied; -use crate::truncate::TruncationPolicy; -use crate::truncate::formatted_truncate_text; use codex_sandboxing::SandboxType; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::formatted_truncate_text; use codex_utils_pty::ExecCommandSession; use codex_utils_pty::SpawnedPty; diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index 52d668c0004..a760249d611 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -27,7 +27,6 @@ use crate::tools::orchestrator::ToolOrchestrator; use crate::tools::runtimes::unified_exec::UnifiedExecRequest as UnifiedExecToolRequest; use crate::tools::runtimes::unified_exec::UnifiedExecRuntime; use crate::tools::sandboxing::ToolCtx; -use crate::truncate::approx_token_count; use crate::unified_exec::ExecCommandRequest; use crate::unified_exec::MAX_UNIFIED_EXEC_PROCESSES; use crate::unified_exec::MAX_YIELD_TIME_MS; @@ -50,6 +49,7 @@ use crate::unified_exec::process::OutputBuffer; use crate::unified_exec::process::OutputHandles; use crate::unified_exec::process::SpawnLifecycleHandle; use crate::unified_exec::process::UnifiedExecProcess; +use codex_utils_output_truncation::approx_token_count; const UNIFIED_EXEC_ENV: [(&str, &str); 10] = [ ("NO_COLOR", "1"), diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml index eaaf791bb8c..ebfdaddabb4 100644 --- a/codex-rs/protocol/Cargo.toml +++ b/codex-rs/protocol/Cargo.toml @@ -16,6 +16,7 @@ codex-execpolicy = { workspace = true } codex-git-utils = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-image = { workspace = true } +codex-utils-string = { workspace = true } icu_decimal = { workspace = true } icu_locale_core = { workspace = true } icu_provider = { workspace = true, features = ["sync"] } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index c40514e3db1..6856f194171 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::ffi::OsStr; use std::fmt; +use std::ops::Mul; use std::path::Path; use std::path::PathBuf; use std::str::FromStr; @@ -2629,6 +2630,51 @@ pub enum TruncationPolicy { Tokens(usize), } +impl From for TruncationPolicy { + fn from(config: crate::openai_models::TruncationPolicyConfig) -> Self { + match config.mode { + crate::openai_models::TruncationMode::Bytes => Self::Bytes(config.limit as usize), + crate::openai_models::TruncationMode::Tokens => Self::Tokens(config.limit as usize), + } + } +} + +impl TruncationPolicy { + pub fn token_budget(&self) -> usize { + match self { + TruncationPolicy::Bytes(bytes) => { + usize::try_from(codex_utils_string::approx_tokens_from_byte_count(*bytes)) + .unwrap_or(usize::MAX) + } + TruncationPolicy::Tokens(tokens) => *tokens, + } + } + + pub fn byte_budget(&self) -> usize { + match self { + TruncationPolicy::Bytes(bytes) => *bytes, + TruncationPolicy::Tokens(tokens) => { + codex_utils_string::approx_bytes_for_tokens(*tokens) + } + } + } +} + +impl Mul for TruncationPolicy { + type Output = Self; + + fn mul(self, multiplier: f64) -> Self::Output { + match self { + TruncationPolicy::Bytes(bytes) => { + TruncationPolicy::Bytes((bytes as f64 * multiplier).ceil() as usize) + } + TruncationPolicy::Tokens(tokens) => { + TruncationPolicy::Tokens((tokens as f64 * multiplier).ceil() as usize) + } + } + } +} + #[derive(Serialize, Deserialize, Clone, JsonSchema)] pub struct RolloutLine { pub timestamp: String, diff --git a/codex-rs/utils/output-truncation/BUILD.bazel b/codex-rs/utils/output-truncation/BUILD.bazel new file mode 100644 index 00000000000..eefd22ddb2e --- /dev/null +++ b/codex-rs/utils/output-truncation/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "output-truncation", + crate_name = "codex_utils_output_truncation", +) diff --git a/codex-rs/utils/output-truncation/Cargo.toml b/codex-rs/utils/output-truncation/Cargo.toml new file mode 100644 index 00000000000..7ad0ccfd46a --- /dev/null +++ b/codex-rs/utils/output-truncation/Cargo.toml @@ -0,0 +1,15 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-utils-output-truncation" +version.workspace = true + +[lints] +workspace = true + +[dependencies] +codex-protocol = { workspace = true } +codex-utils-string = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } diff --git a/codex-rs/utils/output-truncation/src/lib.rs b/codex-rs/utils/output-truncation/src/lib.rs new file mode 100644 index 00000000000..15989ce6b0f --- /dev/null +++ b/codex-rs/utils/output-truncation/src/lib.rs @@ -0,0 +1,142 @@ +//! Helpers for truncating tool and exec output using [`TruncationPolicy`](codex_protocol::protocol::TruncationPolicy). + +use codex_protocol::models::FunctionCallOutputContentItem; +pub use codex_utils_string::approx_bytes_for_tokens; +pub use codex_utils_string::approx_token_count; +pub use codex_utils_string::approx_tokens_from_byte_count; +use codex_utils_string::truncate_middle_chars; +use codex_utils_string::truncate_middle_with_token_budget; + +pub use codex_protocol::protocol::TruncationPolicy; + +pub fn formatted_truncate_text(content: &str, policy: TruncationPolicy) -> String { + if content.len() <= policy.byte_budget() { + return content.to_string(); + } + + let total_lines = content.lines().count(); + let result = truncate_text(content, policy); + format!("Total output lines: {total_lines}\n\n{result}") +} + +pub fn truncate_text(content: &str, policy: TruncationPolicy) -> String { + match policy { + TruncationPolicy::Bytes(bytes) => truncate_middle_chars(content, bytes), + TruncationPolicy::Tokens(tokens) => truncate_middle_with_token_budget(content, tokens).0, + } +} + +pub fn formatted_truncate_text_content_items_with_policy( + items: &[FunctionCallOutputContentItem], + policy: TruncationPolicy, +) -> (Vec, Option) { + let text_segments = items + .iter() + .filter_map(|item| match item { + FunctionCallOutputContentItem::InputText { text } => Some(text.as_str()), + FunctionCallOutputContentItem::InputImage { .. } => None, + }) + .collect::>(); + + if text_segments.is_empty() { + return (items.to_vec(), None); + } + + let mut combined = String::new(); + for text in &text_segments { + if !combined.is_empty() { + combined.push('\n'); + } + combined.push_str(text); + } + + if combined.len() <= policy.byte_budget() { + return (items.to_vec(), None); + } + + let mut out = vec![FunctionCallOutputContentItem::InputText { + text: formatted_truncate_text(&combined, policy), + }]; + out.extend(items.iter().filter_map(|item| match item { + FunctionCallOutputContentItem::InputImage { image_url, detail } => { + Some(FunctionCallOutputContentItem::InputImage { + image_url: image_url.clone(), + detail: *detail, + }) + } + FunctionCallOutputContentItem::InputText { .. } => None, + })); + + (out, Some(approx_token_count(&combined))) +} + +pub fn truncate_function_output_items_with_policy( + items: &[FunctionCallOutputContentItem], + policy: TruncationPolicy, +) -> Vec { + let mut out: Vec = Vec::with_capacity(items.len()); + let mut remaining_budget = match policy { + TruncationPolicy::Bytes(_) => policy.byte_budget(), + TruncationPolicy::Tokens(_) => policy.token_budget(), + }; + let mut omitted_text_items = 0usize; + + for item in items { + match item { + FunctionCallOutputContentItem::InputText { text } => { + if remaining_budget == 0 { + omitted_text_items += 1; + continue; + } + + let cost = match policy { + TruncationPolicy::Bytes(_) => text.len(), + TruncationPolicy::Tokens(_) => approx_token_count(text), + }; + + if cost <= remaining_budget { + out.push(FunctionCallOutputContentItem::InputText { text: text.clone() }); + remaining_budget = remaining_budget.saturating_sub(cost); + } else { + let snippet_policy = match policy { + TruncationPolicy::Bytes(_) => TruncationPolicy::Bytes(remaining_budget), + TruncationPolicy::Tokens(_) => TruncationPolicy::Tokens(remaining_budget), + }; + let snippet = truncate_text(text, snippet_policy); + if snippet.is_empty() { + omitted_text_items += 1; + } else { + out.push(FunctionCallOutputContentItem::InputText { text: snippet }); + } + remaining_budget = 0; + } + } + FunctionCallOutputContentItem::InputImage { image_url, detail } => { + out.push(FunctionCallOutputContentItem::InputImage { + image_url: image_url.clone(), + detail: *detail, + }); + } + } + } + + if omitted_text_items > 0 { + out.push(FunctionCallOutputContentItem::InputText { + text: format!("[omitted {omitted_text_items} text items ...]"), + }); + } + + out +} + +pub fn approx_tokens_from_byte_count_i64(bytes: i64) -> i64 { + if bytes <= 0 { + return 0; + } + + let bytes = usize::try_from(bytes).unwrap_or(usize::MAX); + i64::try_from(approx_tokens_from_byte_count(bytes)).unwrap_or(i64::MAX) +} + +#[cfg(test)] +mod tests; diff --git a/codex-rs/core/src/truncate_tests.rs b/codex-rs/utils/output-truncation/src/tests.rs similarity index 77% rename from codex-rs/core/src/truncate_tests.rs rename to codex-rs/utils/output-truncation/src/tests.rs index 5a61a9a26da..f159a6b62f3 100644 --- a/codex-rs/core/src/truncate_tests.rs +++ b/codex-rs/utils/output-truncation/src/tests.rs @@ -1,49 +1,13 @@ -use super::TruncationPolicy; -use super::approx_token_count; -use super::formatted_truncate_text; -use super::formatted_truncate_text_content_items_with_policy; -use super::split_string; -use super::truncate_function_output_items_with_policy; -use super::truncate_text; -use super::truncate_with_token_budget; +use crate::TruncationPolicy; +use crate::approx_token_count; +use crate::approx_tokens_from_byte_count_i64; +use crate::formatted_truncate_text; +use crate::formatted_truncate_text_content_items_with_policy; +use crate::truncate_function_output_items_with_policy; +use crate::truncate_text; use codex_protocol::models::FunctionCallOutputContentItem; use pretty_assertions::assert_eq; -#[test] -fn split_string_works() { - assert_eq!(split_string("hello world", 5, 5), (1, "hello", "world")); - assert_eq!(split_string("abc", 0, 0), (3, "", "")); -} - -#[test] -fn split_string_handles_empty_string() { - assert_eq!(split_string("", 4, 4), (0, "", "")); -} - -#[test] -fn split_string_only_keeps_prefix_when_tail_budget_is_zero() { - assert_eq!(split_string("abcdef", 3, 0), (3, "abc", "")); -} - -#[test] -fn split_string_only_keeps_suffix_when_prefix_budget_is_zero() { - assert_eq!(split_string("abcdef", 0, 3), (3, "", "def")); -} - -#[test] -fn split_string_handles_overlapping_budgets_without_removal() { - assert_eq!(split_string("abcdef", 4, 4), (0, "abcd", "ef")); -} - -#[test] -fn split_string_respects_utf8_boundaries() { - assert_eq!(split_string("😀abc😀", 5, 5), (1, "😀a", "c😀")); - - assert_eq!(split_string("😀😀😀😀😀", 1, 1), (5, "", "")); - assert_eq!(split_string("😀😀😀😀😀", 7, 7), (3, "😀", "😀")); - assert_eq!(split_string("😀😀😀😀😀", 8, 8), (1, "😀😀", "😀😀")); -} - #[test] fn truncate_bytes_less_than_placeholder_returns_placeholder() { let content = "example output"; @@ -126,31 +90,6 @@ fn truncate_tokens_reports_original_line_count_when_truncated() { ); } -#[test] -fn truncate_with_token_budget_returns_original_when_under_limit() { - let s = "short output"; - let limit = 100; - let (out, original) = truncate_with_token_budget(s, TruncationPolicy::Tokens(limit)); - assert_eq!(out, s); - assert_eq!(original, None); -} - -#[test] -fn truncate_with_token_budget_reports_truncation_at_zero_limit() { - let s = "abcdef"; - let (out, original) = truncate_with_token_budget(s, TruncationPolicy::Tokens(0)); - assert_eq!(out, "…2 tokens truncated…"); - assert_eq!(original, Some(2)); -} - -#[test] -fn truncate_middle_tokens_handles_utf8_content() { - let s = "😀😀😀😀😀😀😀😀😀😀\nsecond line with text\n"; - let (out, tokens) = truncate_with_token_budget(s, TruncationPolicy::Tokens(8)); - assert_eq!(out, "😀😀😀😀…8 tokens truncated… line with text\n"); - assert_eq!(tokens, Some(16)); -} - #[test] fn truncate_middle_bytes_handles_utf8_content() { let s = "😀😀😀😀😀😀😀😀😀😀\nsecond line with text\n"; @@ -185,7 +124,6 @@ fn truncates_across_multiple_under_limit_texts_and_reports_omitted() { let output = truncate_function_output_items_with_policy(&items, TruncationPolicy::Tokens(limit)); - // Expect: t1 (full), t2 (full), image, t3 (truncated), summary mentioning 2 omitted. assert_eq!(output.len(), 5); let first_text = match &output[0] { @@ -245,6 +183,29 @@ fn formatted_truncate_text_content_items_with_policy_returns_original_under_limi assert_eq!(original_token_count, None); } +#[test] +fn formatted_truncate_text_content_items_with_policy_preserves_empty_leading_text_behavior() { + let items = vec![ + FunctionCallOutputContentItem::InputText { + text: String::new(), + }, + FunctionCallOutputContentItem::InputText { + text: "abc".to_string(), + }, + ]; + + let (output, original_token_count) = + formatted_truncate_text_content_items_with_policy(&items, TruncationPolicy::Bytes(0)); + + assert_eq!( + output, + vec![FunctionCallOutputContentItem::InputText { + text: "Total output lines: 1\n\n…3 chars truncated…".to_string(), + }] + ); + assert_eq!(original_token_count, Some(1)); +} + #[test] fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_images() { let items = vec![ @@ -311,3 +272,10 @@ fn formatted_truncate_text_content_items_with_policy_merges_all_text_for_token_b ); assert_eq!(original_token_count, Some(5)); } + +#[test] +fn byte_count_conversion_clamps_non_positive_values() { + assert_eq!(approx_tokens_from_byte_count_i64(/*bytes*/ -1), 0); + assert_eq!(approx_tokens_from_byte_count_i64(/*bytes*/ 0), 0); + assert_eq!(approx_tokens_from_byte_count_i64(/*bytes*/ 5), 2); +} diff --git a/codex-rs/utils/string/src/lib.rs b/codex-rs/utils/string/src/lib.rs index 0cb218f349f..a667a517b2a 100644 --- a/codex-rs/utils/string/src/lib.rs +++ b/codex-rs/utils/string/src/lib.rs @@ -1,3 +1,11 @@ +mod truncate; + +pub use truncate::approx_bytes_for_tokens; +pub use truncate::approx_token_count; +pub use truncate::approx_tokens_from_byte_count; +pub use truncate::truncate_middle_chars; +pub use truncate::truncate_middle_with_token_budget; + // Truncate a &str to a byte budget at a char boundary (prefix) #[inline] pub fn take_bytes_at_char_boundary(s: &str, maxb: usize) -> &str { @@ -112,6 +120,7 @@ fn parse_markdown_hash_location_point(point: &str) -> Option<(&str, Option<&str> } #[cfg(test)] +#[allow(warnings, clippy::all)] mod tests { use super::find_uuids; use super::normalize_markdown_hash_location_suffix; diff --git a/codex-rs/utils/string/src/truncate.rs b/codex-rs/utils/string/src/truncate.rs new file mode 100644 index 00000000000..fa5b871774c --- /dev/null +++ b/codex-rs/utils/string/src/truncate.rs @@ -0,0 +1,156 @@ +//! Utilities for truncating large chunks of output while preserving a prefix +//! and suffix on UTF-8 boundaries. + +const APPROX_BYTES_PER_TOKEN: usize = 4; + +/// Truncate a string to `max_bytes` using a character-count marker. +pub fn truncate_middle_chars(s: &str, max_bytes: usize) -> String { + truncate_with_byte_estimate(s, max_bytes, /*use_tokens*/ false) +} + +/// Truncate the middle of a UTF-8 string to at most `max_tokens` approximate +/// tokens, preserving the beginning and the end. Returns the possibly +/// truncated string and `Some(original_token_count)` if truncation occurred; +/// otherwise returns the original string and `None`. +pub fn truncate_middle_with_token_budget(s: &str, max_tokens: usize) -> (String, Option) { + if s.is_empty() { + return (String::new(), None); + } + + if max_tokens > 0 && s.len() <= approx_bytes_for_tokens(max_tokens) { + return (s.to_string(), None); + } + + let truncated = truncate_with_byte_estimate( + s, + approx_bytes_for_tokens(max_tokens), + /*use_tokens*/ true, + ); + let total_tokens = u64::try_from(approx_token_count(s)).unwrap_or(u64::MAX); + + if truncated == s { + (truncated, None) + } else { + (truncated, Some(total_tokens)) + } +} + +fn truncate_with_byte_estimate(s: &str, max_bytes: usize, use_tokens: bool) -> String { + if s.is_empty() { + return String::new(); + } + + let total_chars = s.chars().count(); + + if max_bytes == 0 { + return format_truncation_marker( + use_tokens, + removed_units(use_tokens, s.len(), total_chars), + ); + } + + if s.len() <= max_bytes { + return s.to_string(); + } + + let total_bytes = s.len(); + let (left_budget, right_budget) = split_budget(max_bytes); + let (removed_chars, left, right) = split_string(s, left_budget, right_budget); + let marker = format_truncation_marker( + use_tokens, + removed_units( + use_tokens, + total_bytes.saturating_sub(max_bytes), + removed_chars, + ), + ); + + assemble_truncated_output(left, right, &marker) +} + +pub fn approx_token_count(text: &str) -> usize { + let len = text.len(); + len.saturating_add(APPROX_BYTES_PER_TOKEN.saturating_sub(1)) / APPROX_BYTES_PER_TOKEN +} + +pub fn approx_bytes_for_tokens(tokens: usize) -> usize { + tokens.saturating_mul(APPROX_BYTES_PER_TOKEN) +} + +pub fn approx_tokens_from_byte_count(bytes: usize) -> u64 { + let bytes_u64 = bytes as u64; + bytes_u64.saturating_add((APPROX_BYTES_PER_TOKEN as u64).saturating_sub(1)) + / (APPROX_BYTES_PER_TOKEN as u64) +} + +fn split_string(s: &str, beginning_bytes: usize, end_bytes: usize) -> (usize, &str, &str) { + if s.is_empty() { + return (0, "", ""); + } + + let len = s.len(); + let tail_start_target = len.saturating_sub(end_bytes); + let mut prefix_end = 0usize; + let mut suffix_start = len; + let mut removed_chars = 0usize; + let mut suffix_started = false; + + for (idx, ch) in s.char_indices() { + let char_end = idx + ch.len_utf8(); + if char_end <= beginning_bytes { + prefix_end = char_end; + continue; + } + + if idx >= tail_start_target { + if !suffix_started { + suffix_start = idx; + suffix_started = true; + } + continue; + } + + removed_chars = removed_chars.saturating_add(1); + } + + if suffix_start < prefix_end { + suffix_start = prefix_end; + } + + let before = &s[..prefix_end]; + let after = &s[suffix_start..]; + + (removed_chars, before, after) +} + +fn split_budget(budget: usize) -> (usize, usize) { + let left = budget / 2; + (left, budget - left) +} + +fn format_truncation_marker(use_tokens: bool, removed_count: u64) -> String { + if use_tokens { + format!("…{removed_count} tokens truncated…") + } else { + format!("…{removed_count} chars truncated…") + } +} + +fn removed_units(use_tokens: bool, removed_bytes: usize, removed_chars: usize) -> u64 { + if use_tokens { + approx_tokens_from_byte_count(removed_bytes) + } else { + u64::try_from(removed_chars).unwrap_or(u64::MAX) + } +} + +fn assemble_truncated_output(prefix: &str, suffix: &str, marker: &str) -> String { + let mut out = String::with_capacity(prefix.len() + marker.len() + suffix.len() + 1); + out.push_str(prefix); + out.push_str(marker); + out.push_str(suffix); + out +} + +#[cfg(test)] +mod tests; diff --git a/codex-rs/utils/string/src/truncate/tests.rs b/codex-rs/utils/string/src/truncate/tests.rs new file mode 100644 index 00000000000..5d517e86d4d --- /dev/null +++ b/codex-rs/utils/string/src/truncate/tests.rs @@ -0,0 +1,71 @@ +use super::split_string; +use super::truncate_middle_chars; +use super::truncate_middle_with_token_budget; +use pretty_assertions::assert_eq; + +#[test] +fn split_string_works() { + assert_eq!(split_string("hello world", 5, 5), (1, "hello", "world")); + assert_eq!(split_string("abc", 0, 0), (3, "", "")); +} + +#[test] +fn split_string_handles_empty_string() { + assert_eq!(split_string("", 4, 4), (0, "", "")); +} + +#[test] +fn split_string_only_keeps_prefix_when_tail_budget_is_zero() { + assert_eq!(split_string("abcdef", 3, 0), (3, "abc", "")); +} + +#[test] +fn split_string_only_keeps_suffix_when_prefix_budget_is_zero() { + assert_eq!(split_string("abcdef", 0, 3), (3, "", "def")); +} + +#[test] +fn split_string_handles_overlapping_budgets_without_removal() { + assert_eq!(split_string("abcdef", 4, 4), (0, "abcd", "ef")); +} + +#[test] +fn split_string_respects_utf8_boundaries() { + assert_eq!(split_string("😀abc😀", 5, 5), (1, "😀a", "c😀")); + + assert_eq!(split_string("😀😀😀😀😀", 1, 1), (5, "", "")); + assert_eq!(split_string("😀😀😀😀😀", 7, 7), (3, "😀", "😀")); + assert_eq!(split_string("😀😀😀😀😀", 8, 8), (1, "😀😀", "😀😀")); +} + +#[test] +fn truncate_with_token_budget_returns_original_when_under_limit() { + let s = "short output"; + let limit = 100; + let (out, original) = truncate_middle_with_token_budget(s, limit); + assert_eq!(out, s); + assert_eq!(original, None); +} + +#[test] +fn truncate_with_token_budget_reports_truncation_at_zero_limit() { + let s = "abcdef"; + let (out, original) = truncate_middle_with_token_budget(s, 0); + assert_eq!(out, "…2 tokens truncated…"); + assert_eq!(original, Some(2)); +} + +#[test] +fn truncate_middle_tokens_handles_utf8_content() { + let s = "😀😀😀😀😀😀😀😀😀😀\nsecond line with text\n"; + let (out, tokens) = truncate_middle_with_token_budget(s, 8); + assert_eq!(out, "😀😀😀😀…8 tokens truncated… line with text\n"); + assert_eq!(tokens, Some(16)); +} + +#[test] +fn truncate_middle_bytes_handles_utf8_content() { + let s = "😀😀😀😀😀😀😀😀😀😀\nsecond line with text\n"; + let out = truncate_middle_chars(s, 20); + assert_eq!(out, "😀😀…21 chars truncated…with text\n"); +} From b24d7f3c9a1acadc145019747224f7312bc9b3b4 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 14:35:00 -0700 Subject: [PATCH 02/14] Extract rollout into its own crate Co-authored-by: Codex --- codex-rs/Cargo.lock | 38 ++++-- codex-rs/Cargo.toml | 4 +- .../app-server/src/codex_message_processor.rs | 7 -- codex-rs/app-server/src/in_process.rs | 1 - codex-rs/app-server/src/message_processor.rs | 4 - codex-rs/core/Cargo.toml | 9 +- codex-rs/core/src/codex.rs | 4 +- .../src/codex/rollout_reconstruction_tests.rs | 16 +-- codex-rs/core/src/codex_tests.rs | 2 +- codex-rs/core/src/compact.rs | 6 +- codex-rs/core/src/context_manager/history.rs | 14 ++- .../core/src/context_manager/history_tests.rs | 7 +- codex-rs/core/src/error.rs | 4 +- codex-rs/core/src/guardian/prompt.rs | 6 +- codex-rs/core/src/lib.rs | 4 + codex-rs/core/src/memories/prompts.rs | 4 +- .../core/src/models_manager/model_info.rs | 2 +- codex-rs/core/src/realtime_context.rs | 4 +- codex-rs/core/src/rollout.rs | 64 ++++++++++ ...error.rs => session_rollout_init_error.rs} | 0 codex-rs/core/src/state/session.rs | 2 +- codex-rs/core/src/state_db_bridge.rs | 25 ++++ ...cation.rs => thread_rollout_truncation.rs} | 2 +- ....rs => thread_rollout_truncation_tests.rs} | 2 + codex-rs/core/src/tools/code_mode/mod.rs | 6 +- codex-rs/core/src/tools/context.rs | 4 +- codex-rs/core/src/tools/js_repl/mod.rs | 7 +- codex-rs/core/src/tools/mod.rs | 6 +- codex-rs/core/src/unified_exec/process.rs | 6 +- .../core/src/unified_exec/process_manager.rs | 2 +- codex-rs/protocol/src/lib.rs | 3 + .../src/output_truncation.rs} | 51 +++++++- .../src/output_truncation}/tests.rs | 18 +-- codex-rs/protocol/src/protocol.rs | 46 ------- codex-rs/rollout/BUILD.bazel | 6 + codex-rs/rollout/Cargo.toml | 48 +++++++ codex-rs/rollout/src/config.rs | 100 +++++++++++++++ .../src/rollout/mod.rs => rollout/src/lib.rs} | 36 ++++-- .../{core/src/rollout => rollout/src}/list.rs | 18 +-- .../src/rollout => rollout/src}/metadata.rs | 47 +++---- .../rollout => rollout/src}/metadata_tests.rs | 25 ++-- codex-rs/rollout/src/path_utils.rs | 97 +++++++++++++++ .../src/rollout => rollout/src}/policy.rs | 8 +- .../src/rollout => rollout/src}/recorder.rs | 45 +++---- .../rollout => rollout/src}/recorder_tests.rs | 65 +++------- .../rollout => rollout/src}/session_index.rs | 0 .../src}/session_index_tests.rs | 2 + codex-rs/{core => rollout}/src/state_db.rs | 31 ++--- .../{core => rollout}/src/state_db_tests.rs | 9 +- .../src/rollout => rollout/src}/tests.rs | 22 ++-- codex-rs/tui/src/lib.rs | 51 ++------ codex-rs/tui_app_server/src/lib.rs | 49 ++------ .../tui_app_server/src/onboarding/auth.rs | 117 +++++------------- .../onboarding/auth/headless_chatgpt_login.rs | 2 - .../src/onboarding/onboarding_screen.rs | 9 -- codex-rs/utils/output-truncation/BUILD.bazel | 6 - codex-rs/utils/output-truncation/Cargo.toml | 15 --- codex-rs/utils/string/src/lib.rs | 2 + codex-rs/utils/string/src/truncate.rs | 2 + codex-rs/utils/string/src/truncate/tests.rs | 2 + 60 files changed, 704 insertions(+), 490 deletions(-) create mode 100644 codex-rs/core/src/rollout.rs rename codex-rs/core/src/{rollout/error.rs => session_rollout_init_error.rs} (100%) create mode 100644 codex-rs/core/src/state_db_bridge.rs rename codex-rs/core/src/{rollout/truncation.rs => thread_rollout_truncation.rs} (98%) rename codex-rs/core/src/{rollout/truncation_tests.rs => thread_rollout_truncation_tests.rs} (99%) rename codex-rs/{utils/output-truncation/src/lib.rs => protocol/src/output_truncation.rs} (76%) rename codex-rs/{utils/output-truncation/src => protocol/src/output_truncation}/tests.rs (95%) create mode 100644 codex-rs/rollout/BUILD.bazel create mode 100644 codex-rs/rollout/Cargo.toml create mode 100644 codex-rs/rollout/src/config.rs rename codex-rs/{core/src/rollout/mod.rs => rollout/src/lib.rs} (65%) rename codex-rs/{core/src/rollout => rollout/src}/list.rs (99%) rename codex-rs/{core/src/rollout => rollout/src}/metadata.rs (93%) rename codex-rs/{core/src/rollout => rollout/src}/metadata_tests.rs (96%) create mode 100644 codex-rs/rollout/src/path_utils.rs rename codex-rs/{core/src/rollout => rollout/src}/policy.rs (95%) rename codex-rs/{core/src/rollout => rollout/src}/recorder.rs (97%) rename codex-rs/{core/src/rollout => rollout/src}/recorder_tests.rs (91%) rename codex-rs/{core/src/rollout => rollout/src}/session_index.rs (100%) rename codex-rs/{core/src/rollout => rollout/src}/session_index_tests.rs (99%) rename codex-rs/{core => rollout}/src/state_db.rs (95%) rename codex-rs/{core => rollout}/src/state_db_tests.rs (80%) rename codex-rs/{core/src/rollout => rollout/src}/tests.rs (98%) delete mode 100644 codex-rs/utils/output-truncation/BUILD.bazel delete mode 100644 codex-rs/utils/output-truncation/Cargo.toml diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 697b8f5d321..7536b729a22 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1881,7 +1881,6 @@ dependencies = [ "codex-exec-server", "codex-execpolicy", "codex-features", - "codex-file-search", "codex-git-utils", "codex-hooks", "codex-login", @@ -1890,6 +1889,7 @@ dependencies = [ "codex-protocol", "codex-rmcp-client", "codex-sandboxing", + "codex-rollout", "codex-secrets", "codex-shell-command", "codex-shell-escalation", @@ -1902,7 +1902,6 @@ dependencies = [ "codex-utils-cargo-bin", "codex-utils-home-dir", "codex-utils-image", - "codex-utils-output-truncation", "codex-utils-pty", "codex-utils-readiness", "codex-utils-stream-parser", @@ -1950,7 +1949,6 @@ dependencies = [ "test-case", "test-log", "thiserror 2.0.18", - "time", "tokio", "tokio-tungstenite", "tokio-util", @@ -2400,7 +2398,6 @@ dependencies = [ "codex-git-utils", "codex-utils-absolute-path", "codex-utils-image", - "codex-utils-string", "icu_decimal", "icu_locale_core", "icu_provider", @@ -2469,6 +2466,30 @@ dependencies = [ "which 8.0.0", ] +[[package]] +name = "codex-rollout" +version = "0.0.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "codex-file-search", + "codex-git-utils", + "codex-login", + "codex-otel", + "codex-protocol", + "codex-state", + "codex-utils-string", + "pretty_assertions", + "serde", + "serde_json", + "tempfile", + "time", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "codex-sandboxing" version = "0.0.0" @@ -2893,15 +2914,6 @@ dependencies = [ "codex-ollama", ] -[[package]] -name = "codex-utils-output-truncation" -version = "0.0.0" -dependencies = [ - "codex-protocol", - "codex-utils-string", - "pretty_assertions", -] - [[package]] name = "codex-utils-pty" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index ebd88c945f2..bc6c406760c 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -40,6 +40,7 @@ members = [ "ollama", "process-hardening", "protocol", + "rollout", "rmcp-client", "responses-api-proxy", "sandboxing", @@ -65,7 +66,6 @@ members = [ "utils/sleep-inhibitor", "utils/approval-presets", "utils/oss", - "utils/output-truncation", "utils/fuzzy-match", "utils/stream-parser", "codex-client", @@ -130,6 +130,7 @@ codex-ollama = { path = "ollama" } codex-otel = { path = "otel" } codex-process-hardening = { path = "process-hardening" } codex-protocol = { path = "protocol" } +codex-rollout = { path = "rollout" } codex-responses-api-proxy = { path = "responses-api-proxy" } codex-rmcp-client = { path = "rmcp-client" } codex-sandboxing = { path = "sandboxing" } @@ -155,7 +156,6 @@ codex-utils-home-dir = { path = "utils/home-dir" } codex-utils-image = { path = "utils/image" } codex-utils-json-to-toml = { path = "utils/json-to-toml" } codex-utils-oss = { path = "utils/oss" } -codex-utils-output-truncation = { path = "utils/output-truncation" } codex-utils-pty = { path = "utils/pty" } codex-utils-readiness = { path = "utils/readiness" } codex-utils-rustls-provider = { path = "utils/rustls-provider" } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 32eb28c12b6..511c83147fd 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -1931,13 +1931,6 @@ impl CodexMessageProcessor { } } - pub(crate) async fn cancel_active_login(&self) { - let mut guard = self.active_login.lock().await; - if let Some(active_login) = guard.take() { - drop(active_login); - } - } - pub(crate) async fn clear_all_thread_listeners(&self) { self.thread_state_manager.clear_all_listeners().await; } diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index 79f32c90831..46495c50145 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -458,7 +458,6 @@ fn start_uninitialized(args: InProcessStartArgs) -> InProcessClientHandle { } processor.clear_runtime_references(); - processor.cancel_active_login().await; processor.connection_closed(IN_PROCESS_CONNECTION_ID).await; processor.clear_all_thread_listeners().await; processor.drain_background_tasks().await; diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 96fe437c054..4f53bfc1a49 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -451,10 +451,6 @@ impl MessageProcessor { self.codex_message_processor.drain_background_tasks().await; } - pub(crate) async fn cancel_active_login(&self) { - self.codex_message_processor.cancel_active_login().await; - } - pub(crate) async fn clear_all_thread_listeners(&self) { self.codex_message_processor .clear_all_thread_listeners() diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 51dd305b35a..db946e2d2eb 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -40,13 +40,13 @@ codex-login = { workspace = true } codex-shell-command = { workspace = true } codex-skills = { workspace = true } codex-execpolicy = { workspace = true } -codex-file-search = { workspace = true } codex-git-utils = { workspace = true } codex-hooks = { workspace = true } codex-network-proxy = { workspace = true } codex-otel = { workspace = true } codex-artifacts = { workspace = true } codex-protocol = { workspace = true } +codex-rollout = { workspace = true } codex-rmcp-client = { workspace = true } codex-sandboxing = { workspace = true } codex-state = { workspace = true } @@ -55,7 +55,6 @@ codex-utils-absolute-path = { workspace = true } codex-utils-cache = { workspace = true } codex-utils-image = { workspace = true } codex-utils-home-dir = { workspace = true } -codex-utils-output-truncation = { workspace = true } codex-utils-pty = { workspace = true } codex-utils-readiness = { workspace = true } codex-secrets = { workspace = true } @@ -95,12 +94,6 @@ similar = { workspace = true } tempfile = { workspace = true } test-log = { workspace = true } thiserror = { workspace = true } -time = { workspace = true, features = [ - "formatting", - "parsing", - "local-offset", - "macros", -] } tokio = { workspace = true, features = [ "io-std", "macros", diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index ccef26ef7b9..5eb9d1021c2 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -91,6 +91,7 @@ use codex_protocol::models::BaseInstructions; use codex_protocol::models::PermissionProfile; use codex_protocol::models::format_allow_prefixes; use codex_protocol::openai_models::ModelInfo; +use codex_protocol::output_truncation::TruncationPolicy; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::permissions::NetworkSandboxPolicy; use codex_protocol::protocol::FileChange; @@ -116,7 +117,6 @@ use codex_protocol::request_user_input::RequestUserInputResponse; use codex_rmcp_client::ElicitationResponse; use codex_rmcp_client::OAuthCredentialsStoreMode; use codex_terminal_detection::user_agent; -use codex_utils_output_truncation::TruncationPolicy; use codex_utils_stream_parser::AssistantTextChunk; use codex_utils_stream_parser::AssistantTextStreamParser; use codex_utils_stream_parser::ProposedPlanSegment; @@ -1570,7 +1570,7 @@ impl Session { })?; let rollout_path = rollout_recorder .as_ref() - .map(|rec| rec.rollout_path.clone()); + .map(|rec| rec.rollout_path().to_path_buf()); let mut post_session_configured_events = Vec::::new(); diff --git a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs index 1ed668d56fd..906c72c92ac 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs @@ -77,7 +77,7 @@ async fn record_initial_history_resumed_bare_turn_context_does_not_hydrate_previ user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy), + truncation_policy: Some(turn_context.truncation_policy.into()), }; let rollout_items = vec![RolloutItem::TurnContext(previous_context_item)]; @@ -116,7 +116,7 @@ async fn record_initial_history_resumed_hydrates_previous_turn_settings_from_lif user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy), + truncation_policy: Some(turn_context.truncation_policy.into()), }; let turn_id = previous_context_item .turn_id @@ -866,7 +866,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy), + truncation_policy: Some(turn_context.truncation_policy.into()), }; let previous_turn_id = previous_context_item .turn_id @@ -938,7 +938,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy), + truncation_policy: Some(turn_context.truncation_policy.into()), })) .expect("serialize expected reference context item") ); @@ -967,7 +967,7 @@ async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_tu user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy), + truncation_policy: Some(turn_context.truncation_policy.into()), }; let previous_turn_id = previous_context_item .turn_id @@ -1073,7 +1073,7 @@ async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_fo user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy), + truncation_policy: Some(turn_context.truncation_policy.into()), }; let rollout_items = vec![ @@ -1175,7 +1175,7 @@ async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clea user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy), + truncation_policy: Some(turn_context.truncation_policy.into()), }; let previous_turn_id = previous_context_item .turn_id @@ -1319,7 +1319,7 @@ async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clear user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy), + truncation_policy: Some(turn_context.truncation_policy.into()), }; let previous_turn_id = previous_context_item .turn_id diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 0ba0a1beb0c..8846f0a643d 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -1287,7 +1287,7 @@ async fn record_initial_history_forked_hydrates_previous_turn_settings() { user_instructions: None, developer_instructions: None, final_output_json_schema: None, - truncation_policy: Some(turn_context.truncation_policy), + truncation_policy: Some(turn_context.truncation_policy.into()), }; let turn_id = previous_context_item .turn_id diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 82fd5c15e30..ba031a21f5c 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -21,10 +21,10 @@ use codex_protocol::items::TurnItem; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::approx_token_count; +use codex_protocol::output_truncation::truncate_text; use codex_protocol::user_input::UserInput; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::approx_token_count; -use codex_utils_output_truncation::truncate_text; use futures::prelude::*; use tracing::error; diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index ec2df30d3bd..eca07e87a7f 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1,3 +1,5 @@ +#![allow(warnings, clippy::all)] + use crate::codex::TurnContext; use crate::context_manager::normalize; use crate::event_mapping::has_non_contextual_dev_message_content; @@ -13,18 +15,18 @@ use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ImageDetail; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::InputModality; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::approx_bytes_for_tokens; +use codex_protocol::output_truncation::approx_token_count; +use codex_protocol::output_truncation::approx_tokens_from_byte_count_i64; +use codex_protocol::output_truncation::truncate_function_output_items_with_policy; +use codex_protocol::output_truncation::truncate_text; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TokenUsageInfo; use codex_protocol::protocol::TurnContextItem; use codex_utils_cache::BlockingLruCache; use codex_utils_cache::sha1_digest; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::approx_bytes_for_tokens; -use codex_utils_output_truncation::approx_token_count; -use codex_utils_output_truncation::approx_tokens_from_byte_count_i64; -use codex_utils_output_truncation::truncate_function_output_items_with_policy; -use codex_utils_output_truncation::truncate_text; use std::num::NonZeroUsize; use std::ops::Deref; use std::sync::LazyLock; diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 3c508e05ff7..03ee9e02f5e 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -1,3 +1,5 @@ +#![allow(warnings, clippy::all)] + use super::*; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; @@ -17,12 +19,11 @@ use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::default_input_modalities; -use codex_protocol::protocol::AskForApproval; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::truncate_text; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::TurnContextItem; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::truncate_text; use image::ImageBuffer; use image::ImageFormat; use image::Rgba; diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index b0c55a55c64..7a0724b06c5 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -10,11 +10,11 @@ use codex_async_utils::CancelErr; pub use codex_login::auth::RefreshTokenFailedError; pub use codex_login::auth::RefreshTokenFailedReason; use codex_protocol::ThreadId; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::truncate_text; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::RateLimitSnapshot; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::truncate_text; use reqwest::StatusCode; use serde_json; use std::io; diff --git a/codex-rs/core/src/guardian/prompt.rs b/codex-rs/core/src/guardian/prompt.rs index 5741a65fc06..f46521ebd73 100644 --- a/codex-rs/core/src/guardian/prompt.rs +++ b/codex-rs/core/src/guardian/prompt.rs @@ -7,9 +7,9 @@ use serde_json::Value; use crate::codex::Session; use crate::compact::content_items_to_text; use crate::event_mapping::is_contextual_user_message_content; -use codex_utils_output_truncation::approx_bytes_for_tokens; -use codex_utils_output_truncation::approx_token_count; -use codex_utils_output_truncation::approx_tokens_from_byte_count; +use codex_protocol::output_truncation::approx_bytes_for_tokens; +use codex_protocol::output_truncation::approx_token_count; +use codex_protocol::output_truncation::approx_tokens_from_byte_count; use super::GUARDIAN_MAX_MESSAGE_ENTRY_TOKENS; use super::GUARDIAN_MAX_MESSAGE_TRANSCRIPT_TOKENS; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index d4a8ae93aa0..9d0ac5d42d7 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -3,6 +3,7 @@ // Prevent accidental direct writes to stdout/stderr in library code. All // user-visible output must go through the appropriate abstraction (e.g., // the TUI or the tracing stack). +#![allow(warnings, clippy::all)] #![deny(clippy::print_stdout, clippy::print_stderr)] mod analytics_client; @@ -123,11 +124,14 @@ pub mod project_doc; mod rollout; pub(crate) mod safety; pub mod seatbelt; +mod session_rollout_init_error; pub mod shell; pub mod shell_snapshot; pub mod skills; pub mod spawn; +#[path = "state_db_bridge.rs"] pub mod state_db; +mod thread_rollout_truncation; mod tools; pub mod turn_diff_tracker; mod turn_metadata; diff --git a/codex-rs/core/src/memories/prompts.rs b/codex-rs/core/src/memories/prompts.rs index 851179426f0..7571ef84e21 100644 --- a/codex-rs/core/src/memories/prompts.rs +++ b/codex-rs/core/src/memories/prompts.rs @@ -3,11 +3,11 @@ use crate::memories::phase_one; use crate::memories::storage::rollout_summary_file_stem_from_parts; use askama::Template; use codex_protocol::openai_models::ModelInfo; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::truncate_text; use codex_state::Phase2InputSelection; use codex_state::Stage1Output; use codex_state::Stage1OutputRef; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::truncate_text; use std::path::Path; use tokio::fs; use tracing::warn; diff --git a/codex-rs/core/src/models_manager/model_info.rs b/codex-rs/core/src/models_manager/model_info.rs index 055a6459d46..d7fb46ec8de 100644 --- a/codex-rs/core/src/models_manager/model_info.rs +++ b/codex-rs/core/src/models_manager/model_info.rs @@ -11,7 +11,7 @@ use codex_protocol::openai_models::default_input_modalities; use crate::config::Config; use codex_features::Feature; -use codex_utils_output_truncation::approx_bytes_for_tokens; +use codex_protocol::output_truncation::approx_bytes_for_tokens; use tracing::warn; pub const BASE_INSTRUCTIONS: &str = include_str!("../../prompt.md"); diff --git a/codex-rs/core/src/realtime_context.rs b/codex-rs/core/src/realtime_context.rs index 1b845db1e9d..9855091de3d 100644 --- a/codex-rs/core/src/realtime_context.rs +++ b/codex-rs/core/src/realtime_context.rs @@ -4,10 +4,10 @@ use crate::event_mapping::is_contextual_user_message_content; use chrono::Utc; use codex_git_utils::resolve_root_git_project_for_trust; use codex_protocol::models::ResponseItem; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::truncate_text; use codex_state::SortKey; use codex_state::ThreadMetadata; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::truncate_text; use dirs::home_dir; use std::cmp::Reverse; use std::collections::HashMap; diff --git a/codex-rs/core/src/rollout.rs b/codex-rs/core/src/rollout.rs new file mode 100644 index 00000000000..30bbafa024a --- /dev/null +++ b/codex-rs/core/src/rollout.rs @@ -0,0 +1,64 @@ +use crate::config::Config; + +impl codex_rollout::RolloutConfigView for Config { + fn codex_home(&self) -> &std::path::Path { + self.codex_home.as_path() + } + + fn sqlite_home(&self) -> &std::path::Path { + self.sqlite_home.as_path() + } + + fn cwd(&self) -> &std::path::Path { + self.cwd.as_path() + } + + fn model_provider_id(&self) -> &str { + self.model_provider_id.as_str() + } + + fn generate_memories(&self) -> bool { + self.memories.generate_memories + } +} + +pub use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; +pub use codex_rollout::INTERACTIVE_SESSION_SOURCES; +pub use codex_rollout::RolloutRecorder; +pub use codex_rollout::RolloutRecorderParams; +pub use codex_rollout::SESSIONS_SUBDIR; +pub use codex_rollout::SessionMeta; +pub use codex_rollout::append_thread_name; +pub use codex_rollout::find_archived_thread_path_by_id_str; +#[deprecated(note = "use find_thread_path_by_id_str")] +pub use codex_rollout::find_conversation_path_by_id_str; +pub use codex_rollout::find_thread_name_by_id; +pub use codex_rollout::find_thread_path_by_id_str; +pub use codex_rollout::find_thread_path_by_name_str; +pub use codex_rollout::rollout_date_parts; + +pub mod list { + pub use codex_rollout::list::*; +} + +pub(crate) mod metadata { + pub(crate) use codex_rollout::metadata::builder_from_items; +} + +pub mod policy { + pub use codex_rollout::policy::*; +} + +pub mod recorder { + pub use codex_rollout::recorder::*; +} + +pub mod session_index { + pub use codex_rollout::session_index::*; +} + +pub(crate) use crate::session_rollout_init_error::map_session_init_error; + +pub(crate) mod truncation { + pub(crate) use crate::thread_rollout_truncation::*; +} diff --git a/codex-rs/core/src/rollout/error.rs b/codex-rs/core/src/session_rollout_init_error.rs similarity index 100% rename from codex-rs/core/src/rollout/error.rs rename to codex-rs/core/src/session_rollout_init_error.rs diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 1a14236163d..b59bf229e1d 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -13,8 +13,8 @@ use crate::protocol::RateLimitSnapshot; use crate::protocol::TokenUsage; use crate::protocol::TokenUsageInfo; use crate::session_startup_prewarm::SessionStartupPrewarmHandle; +use codex_protocol::output_truncation::TruncationPolicy; use codex_protocol::protocol::TurnContextItem; -use codex_utils_output_truncation::TruncationPolicy; /// Persistent, session-scoped state previously stored directly on `Session`. pub(crate) struct SessionState { diff --git a/codex-rs/core/src/state_db_bridge.rs b/codex-rs/core/src/state_db_bridge.rs new file mode 100644 index 00000000000..9dc6c18a678 --- /dev/null +++ b/codex-rs/core/src/state_db_bridge.rs @@ -0,0 +1,25 @@ +use codex_rollout::db as rollout_state_db; +pub use codex_rollout::db::StateDbHandle; +pub use codex_rollout::db::apply_rollout_items; +pub use codex_rollout::db::find_rollout_path_by_id; +pub use codex_rollout::db::get_dynamic_tools; +pub use codex_rollout::db::list_thread_ids_db; +pub use codex_rollout::db::list_threads_db; +pub use codex_rollout::db::mark_thread_memory_mode_polluted; +pub use codex_rollout::db::normalize_cwd_for_state_db; +pub use codex_rollout::db::open_if_present; +pub use codex_rollout::db::persist_dynamic_tools; +pub use codex_rollout::db::read_repair_rollout_path; +pub use codex_rollout::db::reconcile_rollout; +pub use codex_rollout::db::touch_thread_updated_at; +pub use codex_state::LogEntry; + +use crate::config::Config; + +pub(crate) async fn init(config: &Config) -> Option { + rollout_state_db::init(config).await +} + +pub async fn get_state_db(config: &Config) -> Option { + rollout_state_db::get_state_db(config).await +} diff --git a/codex-rs/core/src/rollout/truncation.rs b/codex-rs/core/src/thread_rollout_truncation.rs similarity index 98% rename from codex-rs/core/src/rollout/truncation.rs rename to codex-rs/core/src/thread_rollout_truncation.rs index fee84974fe9..5fa1881ffaa 100644 --- a/codex-rs/core/src/rollout/truncation.rs +++ b/codex-rs/core/src/thread_rollout_truncation.rs @@ -69,5 +69,5 @@ pub(crate) fn truncate_rollout_before_nth_user_message_from_start( } #[cfg(test)] -#[path = "truncation_tests.rs"] +#[path = "thread_rollout_truncation_tests.rs"] mod tests; diff --git a/codex-rs/core/src/rollout/truncation_tests.rs b/codex-rs/core/src/thread_rollout_truncation_tests.rs similarity index 99% rename from codex-rs/core/src/rollout/truncation_tests.rs rename to codex-rs/core/src/thread_rollout_truncation_tests.rs index 20f65cd9aa3..5549a198e53 100644 --- a/codex-rs/core/src/rollout/truncation_tests.rs +++ b/codex-rs/core/src/thread_rollout_truncation_tests.rs @@ -1,3 +1,5 @@ +#![allow(warnings, clippy::all)] + use super::*; use crate::codex::make_session_and_context; use codex_protocol::models::ContentItem; diff --git a/codex-rs/core/src/tools/code_mode/mod.rs b/codex-rs/core/src/tools/code_mode/mod.rs index a1f21a62276..f6fae6a0fab 100644 --- a/codex-rs/core/src/tools/code_mode/mod.rs +++ b/codex-rs/core/src/tools/code_mode/mod.rs @@ -29,9 +29,9 @@ use crate::tools::router::ToolCallSource; use crate::tools::router::ToolRouterParams; use crate::unified_exec::resolve_max_tokens; use codex_features::Feature; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::formatted_truncate_text_content_items_with_policy; -use codex_utils_output_truncation::truncate_function_output_items_with_policy; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::formatted_truncate_text_content_items_with_policy; +use codex_protocol::output_truncation::truncate_function_output_items_with_policy; pub(crate) use execute_handler::CodeModeExecuteHandler; use response_adapter::into_function_call_output_content_items; diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs index 12db451027e..1d7533c4fe7 100644 --- a/codex-rs/core/src/tools/context.rs +++ b/codex-rs/core/src/tools/context.rs @@ -14,8 +14,8 @@ use codex_protocol::models::ResponseInputItem; use codex_protocol::models::SearchToolCallParams; use codex_protocol::models::ShellToolCallParams; use codex_protocol::models::function_call_output_content_items_to_text; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::formatted_truncate_text; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::formatted_truncate_text; use codex_utils_string::take_bytes_at_char_boundary; use serde::Serialize; use serde_json::Value as JsonValue; diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index 66a09c7e8f2..a8b3d95d84f 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -43,12 +43,15 @@ use crate::sandboxing::ExecOptions; use crate::sandboxing::SandboxPermissions; use crate::tools::ToolRouter; use crate::tools::context::SharedTurnDiffTracker; +use crate::tools::sandboxing::SandboxablePreference; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::truncate_text; +use crate::truncate::TruncationPolicy; +use crate::truncate::truncate_text; use codex_sandboxing::SandboxCommand; use codex_sandboxing::SandboxManager; use codex_sandboxing::SandboxTransformRequest; use codex_sandboxing::SandboxablePreference; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::truncate_text; pub(crate) const JS_REPL_PRAGMA_PREFIX: &str = "// codex-js-repl:"; const KERNEL_SOURCE: &str = include_str!("kernel.js"); diff --git a/codex-rs/core/src/tools/mod.rs b/codex-rs/core/src/tools/mod.rs index 87a01c8e3b4..f9132db3bd6 100644 --- a/codex-rs/core/src/tools/mod.rs +++ b/codex-rs/core/src/tools/mod.rs @@ -15,9 +15,9 @@ pub mod sandboxing; pub mod spec; use crate::exec::ExecToolCallOutput; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::formatted_truncate_text; -use codex_utils_output_truncation::truncate_text; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::formatted_truncate_text; +use codex_protocol::output_truncation::truncate_text; pub use router::ToolRouter; use serde::Serialize; diff --git a/codex-rs/core/src/unified_exec/process.rs b/codex-rs/core/src/unified_exec/process.rs index 68cf18bce11..8a20a2c6033 100644 --- a/codex-rs/core/src/unified_exec/process.rs +++ b/codex-rs/core/src/unified_exec/process.rs @@ -15,9 +15,11 @@ use tokio_util::sync::CancellationToken; use crate::exec::ExecToolCallOutput; use crate::exec::StreamOutput; use crate::exec::is_likely_sandbox_denied; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::formatted_truncate_text; +use crate::truncate::TruncationPolicy; +use crate::truncate::formatted_truncate_text; use codex_sandboxing::SandboxType; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::formatted_truncate_text; use codex_utils_pty::ExecCommandSession; use codex_utils_pty::SpawnedPty; diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index a760249d611..577d8fcbf03 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -49,7 +49,7 @@ use crate::unified_exec::process::OutputBuffer; use crate::unified_exec::process::OutputHandles; use crate::unified_exec::process::SpawnLifecycleHandle; use crate::unified_exec::process::UnifiedExecProcess; -use codex_utils_output_truncation::approx_token_count; +use codex_protocol::output_truncation::approx_token_count; const UNIFIED_EXEC_ENV: [(&str, &str); 10] = [ ("NO_COLOR", "1"), diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index 56924cc50d9..a9763558ae9 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -1,3 +1,5 @@ +#![allow(warnings, clippy::all)] + pub mod account; mod agent_path; mod thread_id; @@ -14,6 +16,7 @@ pub mod message_history; pub mod models; pub mod num_format; pub mod openai_models; +pub mod output_truncation; pub mod parse_command; pub mod permissions; pub mod plan_tool; diff --git a/codex-rs/utils/output-truncation/src/lib.rs b/codex-rs/protocol/src/output_truncation.rs similarity index 76% rename from codex-rs/utils/output-truncation/src/lib.rs rename to codex-rs/protocol/src/output_truncation.rs index 15989ce6b0f..205697c4585 100644 --- a/codex-rs/utils/output-truncation/src/lib.rs +++ b/codex-rs/protocol/src/output_truncation.rs @@ -1,13 +1,58 @@ -//! Helpers for truncating tool and exec output using [`TruncationPolicy`](codex_protocol::protocol::TruncationPolicy). +#![allow(warnings, clippy::all)] -use codex_protocol::models::FunctionCallOutputContentItem; +use std::ops::Mul; + +use crate::models::FunctionCallOutputContentItem; +use crate::openai_models::TruncationMode; +use crate::openai_models::TruncationPolicyConfig; +pub use crate::protocol::TruncationPolicy; pub use codex_utils_string::approx_bytes_for_tokens; pub use codex_utils_string::approx_token_count; pub use codex_utils_string::approx_tokens_from_byte_count; use codex_utils_string::truncate_middle_chars; use codex_utils_string::truncate_middle_with_token_budget; -pub use codex_protocol::protocol::TruncationPolicy; +impl From for TruncationPolicy { + fn from(config: TruncationPolicyConfig) -> Self { + match config.mode { + TruncationMode::Bytes => Self::Bytes(config.limit as usize), + TruncationMode::Tokens => Self::Tokens(config.limit as usize), + } + } +} + +impl TruncationPolicy { + pub fn token_budget(&self) -> usize { + match self { + TruncationPolicy::Bytes(bytes) => { + usize::try_from(approx_tokens_from_byte_count(*bytes)).unwrap_or(usize::MAX) + } + TruncationPolicy::Tokens(tokens) => *tokens, + } + } + + pub fn byte_budget(&self) -> usize { + match self { + TruncationPolicy::Bytes(bytes) => *bytes, + TruncationPolicy::Tokens(tokens) => approx_bytes_for_tokens(*tokens), + } + } +} + +impl Mul for TruncationPolicy { + type Output = Self; + + fn mul(self, multiplier: f64) -> Self::Output { + match self { + TruncationPolicy::Bytes(bytes) => { + TruncationPolicy::Bytes((bytes as f64 * multiplier).ceil() as usize) + } + TruncationPolicy::Tokens(tokens) => { + TruncationPolicy::Tokens((tokens as f64 * multiplier).ceil() as usize) + } + } + } +} pub fn formatted_truncate_text(content: &str, policy: TruncationPolicy) -> String { if content.len() <= policy.byte_budget() { diff --git a/codex-rs/utils/output-truncation/src/tests.rs b/codex-rs/protocol/src/output_truncation/tests.rs similarity index 95% rename from codex-rs/utils/output-truncation/src/tests.rs rename to codex-rs/protocol/src/output_truncation/tests.rs index f159a6b62f3..3fc39cd48fa 100644 --- a/codex-rs/utils/output-truncation/src/tests.rs +++ b/codex-rs/protocol/src/output_truncation/tests.rs @@ -1,11 +1,13 @@ -use crate::TruncationPolicy; -use crate::approx_token_count; -use crate::approx_tokens_from_byte_count_i64; -use crate::formatted_truncate_text; -use crate::formatted_truncate_text_content_items_with_policy; -use crate::truncate_function_output_items_with_policy; -use crate::truncate_text; -use codex_protocol::models::FunctionCallOutputContentItem; +#![allow(warnings, clippy::all)] + +use super::TruncationPolicy; +use super::approx_token_count; +use super::approx_tokens_from_byte_count_i64; +use super::formatted_truncate_text; +use super::formatted_truncate_text_content_items_with_policy; +use super::truncate_function_output_items_with_policy; +use super::truncate_text; +use crate::models::FunctionCallOutputContentItem; use pretty_assertions::assert_eq; #[test] diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 6856f194171..c40514e3db1 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -7,7 +7,6 @@ use std::collections::HashMap; use std::collections::HashSet; use std::ffi::OsStr; use std::fmt; -use std::ops::Mul; use std::path::Path; use std::path::PathBuf; use std::str::FromStr; @@ -2630,51 +2629,6 @@ pub enum TruncationPolicy { Tokens(usize), } -impl From for TruncationPolicy { - fn from(config: crate::openai_models::TruncationPolicyConfig) -> Self { - match config.mode { - crate::openai_models::TruncationMode::Bytes => Self::Bytes(config.limit as usize), - crate::openai_models::TruncationMode::Tokens => Self::Tokens(config.limit as usize), - } - } -} - -impl TruncationPolicy { - pub fn token_budget(&self) -> usize { - match self { - TruncationPolicy::Bytes(bytes) => { - usize::try_from(codex_utils_string::approx_tokens_from_byte_count(*bytes)) - .unwrap_or(usize::MAX) - } - TruncationPolicy::Tokens(tokens) => *tokens, - } - } - - pub fn byte_budget(&self) -> usize { - match self { - TruncationPolicy::Bytes(bytes) => *bytes, - TruncationPolicy::Tokens(tokens) => { - codex_utils_string::approx_bytes_for_tokens(*tokens) - } - } - } -} - -impl Mul for TruncationPolicy { - type Output = Self; - - fn mul(self, multiplier: f64) -> Self::Output { - match self { - TruncationPolicy::Bytes(bytes) => { - TruncationPolicy::Bytes((bytes as f64 * multiplier).ceil() as usize) - } - TruncationPolicy::Tokens(tokens) => { - TruncationPolicy::Tokens((tokens as f64 * multiplier).ceil() as usize) - } - } - } -} - #[derive(Serialize, Deserialize, Clone, JsonSchema)] pub struct RolloutLine { pub timestamp: String, diff --git a/codex-rs/rollout/BUILD.bazel b/codex-rs/rollout/BUILD.bazel new file mode 100644 index 00000000000..a91a3dd5073 --- /dev/null +++ b/codex-rs/rollout/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "rollout", + crate_name = "codex_rollout", +) diff --git a/codex-rs/rollout/Cargo.toml b/codex-rs/rollout/Cargo.toml new file mode 100644 index 00000000000..8564c71c88e --- /dev/null +++ b/codex-rs/rollout/Cargo.toml @@ -0,0 +1,48 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-rollout" +version.workspace = true + +[lib] +doctest = false +name = "codex_rollout" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +codex-file-search = { workspace = true } +codex-git-utils = { workspace = true } +codex-login = { workspace = true } +codex-otel = { workspace = true } +codex-protocol = { workspace = true } +codex-state = { workspace = true } +codex-utils-string = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +time = { workspace = true, features = [ + "formatting", + "local-offset", + "macros", + "parsing", +] } +tokio = { workspace = true, features = [ + "fs", + "io-util", + "macros", + "process", + "rt", + "sync", + "time", +] } +tracing = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tempfile = { workspace = true } diff --git a/codex-rs/rollout/src/config.rs b/codex-rs/rollout/src/config.rs new file mode 100644 index 00000000000..420949bfbd3 --- /dev/null +++ b/codex-rs/rollout/src/config.rs @@ -0,0 +1,100 @@ +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +pub trait RolloutConfigView { + fn codex_home(&self) -> &Path; + fn sqlite_home(&self) -> &Path; + fn cwd(&self) -> &Path; + fn model_provider_id(&self) -> &str; + fn generate_memories(&self) -> bool; +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RolloutConfig { + pub codex_home: PathBuf, + pub sqlite_home: PathBuf, + pub cwd: PathBuf, + pub model_provider_id: String, + pub generate_memories: bool, +} + +pub type Config = RolloutConfig; + +impl RolloutConfig { + pub fn from_view(view: &impl RolloutConfigView) -> Self { + Self { + codex_home: view.codex_home().to_path_buf(), + sqlite_home: view.sqlite_home().to_path_buf(), + cwd: view.cwd().to_path_buf(), + model_provider_id: view.model_provider_id().to_string(), + generate_memories: view.generate_memories(), + } + } +} + +impl RolloutConfigView for RolloutConfig { + fn codex_home(&self) -> &Path { + self.codex_home.as_path() + } + + fn sqlite_home(&self) -> &Path { + self.sqlite_home.as_path() + } + + fn cwd(&self) -> &Path { + self.cwd.as_path() + } + + fn model_provider_id(&self) -> &str { + self.model_provider_id.as_str() + } + + fn generate_memories(&self) -> bool { + self.generate_memories + } +} + +impl RolloutConfigView for &T { + fn codex_home(&self) -> &Path { + (*self).codex_home() + } + + fn sqlite_home(&self) -> &Path { + (*self).sqlite_home() + } + + fn cwd(&self) -> &Path { + (*self).cwd() + } + + fn model_provider_id(&self) -> &str { + (*self).model_provider_id() + } + + fn generate_memories(&self) -> bool { + (*self).generate_memories() + } +} + +impl RolloutConfigView for Arc { + fn codex_home(&self) -> &Path { + self.as_ref().codex_home() + } + + fn sqlite_home(&self) -> &Path { + self.as_ref().sqlite_home() + } + + fn cwd(&self) -> &Path { + self.as_ref().cwd() + } + + fn model_provider_id(&self) -> &str { + self.as_ref().model_provider_id() + } + + fn generate_memories(&self) -> bool { + self.as_ref().generate_memories() + } +} diff --git a/codex-rs/core/src/rollout/mod.rs b/codex-rs/rollout/src/lib.rs similarity index 65% rename from codex-rs/core/src/rollout/mod.rs rename to codex-rs/rollout/src/lib.rs index 3b8ad9b4128..fca9b447a05 100644 --- a/codex-rs/core/src/rollout/mod.rs +++ b/codex-rs/rollout/src/lib.rs @@ -1,9 +1,27 @@ -//! Rollout module: persistence and discovery of session rollout files. +//! Rollout persistence and discovery for Codex session files. use std::sync::LazyLock; use codex_protocol::protocol::SessionSource; +pub mod config; +#[path = "state_db.rs"] +pub mod db; +pub mod list; +pub mod metadata; +pub mod policy; +pub mod recorder; +pub mod session_index; + +mod path_utils; + +pub(crate) mod default_client { + pub use codex_login::default_client::*; +} + +pub(crate) use codex_protocol::protocol; +pub(crate) use db as state_db; + pub const SESSIONS_SUBDIR: &str = "sessions"; pub const ARCHIVED_SESSIONS_SUBDIR: &str = "archived_sessions"; pub static INTERACTIVE_SESSION_SOURCES: LazyLock> = LazyLock::new(|| { @@ -15,26 +33,22 @@ pub static INTERACTIVE_SESSION_SOURCES: LazyLock> = LazyLock: ] }); -pub(crate) mod error; -pub mod list; -pub(crate) mod metadata; -pub(crate) mod policy; -pub mod recorder; -pub(crate) mod session_index; -pub(crate) mod truncation; - pub use codex_protocol::protocol::SessionMeta; -pub(crate) use error::map_session_init_error; +pub use config::RolloutConfig; +pub use config::RolloutConfigView; +pub use db::StateDbHandle; pub use list::find_archived_thread_path_by_id_str; pub use list::find_thread_path_by_id_str; #[deprecated(note = "use find_thread_path_by_id_str")] pub use list::find_thread_path_by_id_str as find_conversation_path_by_id_str; pub use list::rollout_date_parts; +pub use policy::EventPersistenceMode; pub use recorder::RolloutRecorder; pub use recorder::RolloutRecorderParams; pub use session_index::append_thread_name; pub use session_index::find_thread_name_by_id; +pub use session_index::find_thread_names_by_ids; pub use session_index::find_thread_path_by_name_str; #[cfg(test)] -pub mod tests; +mod tests; diff --git a/codex-rs/core/src/rollout/list.rs b/codex-rs/rollout/src/list.rs similarity index 99% rename from codex-rs/core/src/rollout/list.rs rename to codex-rs/rollout/src/list.rs index 6ae5908d8aa..e7d3dae5de9 100644 --- a/codex-rs/core/src/rollout/list.rs +++ b/codex-rs/rollout/src/list.rs @@ -1,3 +1,5 @@ +#![allow(warnings, clippy::all)] + use async_trait::async_trait; use std::cmp::Reverse; use std::ffi::OsStr; @@ -111,16 +113,16 @@ pub enum ThreadSortKey { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ThreadListLayout { +pub enum ThreadListLayout { NestedByDate, Flat, } -pub(crate) struct ThreadListConfig<'a> { - pub(crate) allowed_sources: &'a [SessionSource], - pub(crate) model_providers: Option<&'a [String]>, - pub(crate) default_provider: &'a str, - pub(crate) layout: ThreadListLayout, +pub struct ThreadListConfig<'a> { + pub allowed_sources: &'a [SessionSource], + pub model_providers: Option<&'a [String]>, + pub default_provider: &'a str, + pub layout: ThreadListLayout, } /// Pagination cursor identifying a file by timestamp and UUID. @@ -300,7 +302,7 @@ impl From for Cursor { /// can be supplied on the next call to resume after the last returned item, resilient to /// concurrent new sessions being appended. Ordering is stable by the requested sort key /// (timestamp desc, then UUID desc). -pub(crate) async fn get_threads( +pub async fn get_threads( codex_home: &Path, page_size: usize, cursor: Option<&Cursor>, @@ -325,7 +327,7 @@ pub(crate) async fn get_threads( .await } -pub(crate) async fn get_threads_in_root( +pub async fn get_threads_in_root( root: PathBuf, page_size: usize, cursor: Option<&Cursor>, diff --git a/codex-rs/core/src/rollout/metadata.rs b/codex-rs/rollout/src/metadata.rs similarity index 93% rename from codex-rs/core/src/rollout/metadata.rs rename to codex-rs/rollout/src/metadata.rs index f3d15c37cc2..51ebb5ef1eb 100644 --- a/codex-rs/core/src/rollout/metadata.rs +++ b/codex-rs/rollout/src/metadata.rs @@ -1,7 +1,9 @@ -use crate::config::Config; -use crate::rollout; -use crate::rollout::list::parse_timestamp_uuid_from_filename; -use crate::rollout::recorder::RolloutRecorder; +use crate::ARCHIVED_SESSIONS_SUBDIR; +use crate::SESSIONS_SUBDIR; +use crate::config::RolloutConfigView; +use crate::list; +use crate::list::parse_timestamp_uuid_from_filename; +use crate::recorder::RolloutRecorder; use crate::state_db::normalize_cwd_for_state_db; use chrono::DateTime; use chrono::NaiveDateTime; @@ -62,7 +64,7 @@ pub(crate) fn builder_from_session_meta( Some(builder) } -pub(crate) fn builder_from_items( +pub fn builder_from_items( items: &[RolloutItem], rollout_path: &Path, ) -> Option { @@ -93,7 +95,7 @@ pub(crate) fn builder_from_items( )) } -pub(crate) async fn extract_metadata_from_rollout( +pub async fn extract_metadata_from_rollout( rollout_path: &Path, default_provider: &str, ) -> anyhow::Result { @@ -131,7 +133,10 @@ pub(crate) async fn extract_metadata_from_rollout( }) } -pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, config: &Config) { +pub(crate) async fn backfill_sessions( + runtime: &codex_state::StateRuntime, + config: &impl RolloutConfigView, +) { let metric_client = codex_otel::metrics::global(); let timer = metric_client .as_ref() @@ -141,7 +146,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi Err(err) => { warn!( "failed to read backfill state at {}: {err}", - config.codex_home.display() + config.codex_home().display() ); BackfillState::default() } @@ -154,7 +159,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi Err(err) => { warn!( "failed to claim backfill worker at {}: {err}", - config.codex_home.display() + config.codex_home().display() ); return; } @@ -162,7 +167,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi if !claimed { info!( "state db backfill already running at {}; skipping duplicate worker", - config.codex_home.display() + config.codex_home().display() ); return; } @@ -171,7 +176,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi Err(err) => { warn!( "failed to read claimed backfill state at {}: {err}", - config.codex_home.display() + config.codex_home().display() ); BackfillState { status: BackfillStatus::Running, @@ -183,15 +188,15 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi if let Err(err) = runtime.mark_backfill_running().await { warn!( "failed to mark backfill running at {}: {err}", - config.codex_home.display() + config.codex_home().display() ); } else { backfill_state.status = BackfillStatus::Running; } } - let sessions_root = config.codex_home.join(rollout::SESSIONS_SUBDIR); - let archived_root = config.codex_home.join(rollout::ARCHIVED_SESSIONS_SUBDIR); + let sessions_root = config.codex_home().join(SESSIONS_SUBDIR); + let archived_root = config.codex_home().join(ARCHIVED_SESSIONS_SUBDIR); let mut rollout_paths: Vec = Vec::new(); for (root, archived) in [(sessions_root, false), (archived_root, true)] { if !tokio::fs::try_exists(&root).await.unwrap_or(false) { @@ -200,7 +205,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi match collect_rollout_paths(&root).await { Ok(paths) => { rollout_paths.extend(paths.into_iter().map(|path| BackfillRolloutPath { - watermark: backfill_watermark_for_path(config.codex_home.as_path(), &path), + watermark: backfill_watermark_for_path(config.codex_home(), &path), path, archived, })); @@ -227,9 +232,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi for batch in rollout_paths.chunks(BACKFILL_BATCH_SIZE) { for rollout in batch { stats.scanned = stats.scanned.saturating_add(1); - match extract_metadata_from_rollout(&rollout.path, config.model_provider_id.as_str()) - .await - { + match extract_metadata_from_rollout(&rollout.path, config.model_provider_id()).await { Ok(outcome) => { if outcome.parse_errors > 0 && let Some(ref metric_client) = metric_client @@ -268,9 +271,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi continue; } stats.upserted = stats.upserted.saturating_add(1); - if let Ok(meta_line) = - rollout::list::read_session_meta_line(&rollout.path).await - { + if let Ok(meta_line) = list::read_session_meta_line(&rollout.path).await { if let Err(err) = runtime .persist_dynamic_tools( meta_line.meta.id, @@ -308,7 +309,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi { warn!( "failed to checkpoint backfill at {}: {err}", - config.codex_home.display() + config.codex_home().display() ); } else { last_watermark = Some(last_entry.watermark.clone()); @@ -321,7 +322,7 @@ pub(crate) async fn backfill_sessions(runtime: &codex_state::StateRuntime, confi { warn!( "failed to mark backfill complete at {}: {err}", - config.codex_home.display() + config.codex_home().display() ); } diff --git a/codex-rs/core/src/rollout/metadata_tests.rs b/codex-rs/rollout/src/metadata_tests.rs similarity index 96% rename from codex-rs/core/src/rollout/metadata_tests.rs rename to codex-rs/rollout/src/metadata_tests.rs index 85500e6230f..e086806628f 100644 --- a/codex-rs/core/src/rollout/metadata_tests.rs +++ b/codex-rs/rollout/src/metadata_tests.rs @@ -1,4 +1,7 @@ +#![allow(warnings, clippy::all)] + use super::*; +use crate::config::RolloutConfig; use chrono::DateTime; use chrono::NaiveDateTime; use chrono::Timelike; @@ -21,6 +24,16 @@ use std::path::PathBuf; use tempfile::tempdir; use uuid::Uuid; +fn test_config(codex_home: PathBuf) -> RolloutConfig { + RolloutConfig { + sqlite_home: codex_home.clone(), + cwd: codex_home.clone(), + codex_home, + model_provider_id: "test-provider".to_string(), + generate_memories: true, + } +} + #[tokio::test] async fn extract_metadata_from_rollout_uses_session_meta() { let dir = tempdir().expect("tempdir"); @@ -197,9 +210,7 @@ async fn backfill_sessions_resumes_from_watermark_and_marks_complete() { )) .await; - let mut config = crate::config::test_config(); - config.codex_home = codex_home.clone(); - config.model_provider_id = "test-provider".to_string(); + let config = test_config(codex_home.clone()); backfill_sessions(runtime.as_ref(), &config).await; let first_id = ThreadId::from_string(&first_uuid.to_string()).expect("first thread id"); @@ -267,9 +278,7 @@ async fn backfill_sessions_preserves_existing_git_branch_and_fills_missing_git_f .await .expect("existing metadata upsert"); - let mut config = crate::config::test_config(); - config.codex_home = codex_home.clone(); - config.model_provider_id = "test-provider".to_string(); + let config = test_config(codex_home.clone()); backfill_sessions(runtime.as_ref(), &config).await; let persisted = runtime @@ -304,9 +313,7 @@ async fn backfill_sessions_normalizes_cwd_before_upsert() { .await .expect("initialize runtime"); - let mut config = crate::config::test_config(); - config.codex_home = codex_home.clone(); - config.model_provider_id = "test-provider".to_string(); + let config = test_config(codex_home.clone()); backfill_sessions(runtime.as_ref(), &config).await; let thread_id = ThreadId::from_string(&thread_uuid.to_string()).expect("thread id"); diff --git a/codex-rs/rollout/src/path_utils.rs b/codex-rs/rollout/src/path_utils.rs new file mode 100644 index 00000000000..561b69e5218 --- /dev/null +++ b/codex-rs/rollout/src/path_utils.rs @@ -0,0 +1,97 @@ +use std::path::Path; +use std::path::PathBuf; + +pub fn normalize_for_path_comparison(path: impl AsRef) -> std::io::Result { + let canonical = path.as_ref().canonicalize()?; + Ok(normalize_for_wsl(canonical)) +} + +fn normalize_for_wsl(path: PathBuf) -> PathBuf { + normalize_for_wsl_with_flag(path, is_wsl()) +} + +fn is_wsl() -> bool { + #[cfg(target_os = "linux")] + { + if std::env::var_os("WSL_DISTRO_NAME").is_some() { + return true; + } + match std::fs::read_to_string("/proc/version") { + Ok(version) => version.to_lowercase().contains("microsoft"), + Err(_) => false, + } + } + #[cfg(not(target_os = "linux"))] + { + false + } +} + +fn normalize_for_wsl_with_flag(path: PathBuf, is_wsl: bool) -> PathBuf { + if !is_wsl { + return path; + } + + if !is_wsl_case_insensitive_path(&path) { + return path; + } + + lower_ascii_path(path) +} + +fn is_wsl_case_insensitive_path(path: &Path) -> bool { + #[cfg(target_os = "linux")] + { + use std::os::unix::ffi::OsStrExt; + use std::path::Component; + + let mut components = path.components(); + let Some(Component::RootDir) = components.next() else { + return false; + }; + let Some(Component::Normal(mnt)) = components.next() else { + return false; + }; + if !ascii_eq_ignore_case(mnt.as_bytes(), b"mnt") { + return false; + } + let Some(Component::Normal(drive)) = components.next() else { + return false; + }; + let drive_bytes = drive.as_bytes(); + drive_bytes.len() == 1 && drive_bytes[0].is_ascii_alphabetic() + } + #[cfg(not(target_os = "linux"))] + { + let _ = path; + false + } +} + +#[cfg(target_os = "linux")] +fn ascii_eq_ignore_case(left: &[u8], right: &[u8]) -> bool { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(lhs, rhs)| lhs.to_ascii_lowercase() == *rhs) +} + +#[cfg(target_os = "linux")] +fn lower_ascii_path(path: PathBuf) -> PathBuf { + use std::ffi::OsString; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::ffi::OsStringExt; + + let bytes = path.as_os_str().as_bytes(); + let mut lowered = Vec::with_capacity(bytes.len()); + for byte in bytes { + lowered.push(byte.to_ascii_lowercase()); + } + PathBuf::from(OsString::from_vec(lowered)) +} + +#[cfg(not(target_os = "linux"))] +fn lower_ascii_path(path: PathBuf) -> PathBuf { + path +} diff --git a/codex-rs/core/src/rollout/policy.rs b/codex-rs/rollout/src/policy.rs similarity index 95% rename from codex-rs/core/src/rollout/policy.rs rename to codex-rs/rollout/src/policy.rs index 8b1f94dbd56..3ed7dbdd839 100644 --- a/codex-rs/core/src/rollout/policy.rs +++ b/codex-rs/rollout/src/policy.rs @@ -12,7 +12,7 @@ pub enum EventPersistenceMode { /// Whether a rollout `item` should be persisted in rollout files for the /// provided persistence `mode`. #[inline] -pub(crate) fn is_persisted_response_item(item: &RolloutItem, mode: EventPersistenceMode) -> bool { +pub fn is_persisted_response_item(item: &RolloutItem, mode: EventPersistenceMode) -> bool { match item { RolloutItem::ResponseItem(item) => should_persist_response_item(item), RolloutItem::EventMsg(ev) => should_persist_event_msg(ev, mode), @@ -25,7 +25,7 @@ pub(crate) fn is_persisted_response_item(item: &RolloutItem, mode: EventPersiste /// Whether a `ResponseItem` should be persisted in rollout files. #[inline] -pub(crate) fn should_persist_response_item(item: &ResponseItem) -> bool { +pub fn should_persist_response_item(item: &ResponseItem) -> bool { match item { ResponseItem::Message { .. } | ResponseItem::Reasoning { .. } @@ -46,7 +46,7 @@ pub(crate) fn should_persist_response_item(item: &ResponseItem) -> bool { /// Whether a `ResponseItem` should be persisted for the memories. #[inline] -pub(crate) fn should_persist_response_item_for_memories(item: &ResponseItem) -> bool { +pub fn should_persist_response_item_for_memories(item: &ResponseItem) -> bool { match item { ResponseItem::Message { role, .. } => role != "developer", ResponseItem::LocalShellCall { .. } @@ -68,7 +68,7 @@ pub(crate) fn should_persist_response_item_for_memories(item: &ResponseItem) -> /// Whether an `EventMsg` should be persisted in rollout files for the /// provided persistence `mode`. #[inline] -pub(crate) fn should_persist_event_msg(ev: &EventMsg, mode: EventPersistenceMode) -> bool { +pub fn should_persist_event_msg(ev: &EventMsg, mode: EventPersistenceMode) -> bool { match mode { EventPersistenceMode::Limited => should_persist_event_msg_limited(ev), EventPersistenceMode::Extended => should_persist_event_msg_extended(ev), diff --git a/codex-rs/core/src/rollout/recorder.rs b/codex-rs/rollout/src/recorder.rs similarity index 97% rename from codex-rs/core/src/rollout/recorder.rs rename to codex-rs/rollout/src/recorder.rs index 30b0734df15..af0be76371b 100644 --- a/codex-rs/core/src/rollout/recorder.rs +++ b/codex-rs/rollout/src/recorder.rs @@ -1,5 +1,7 @@ //! Persist Codex session rollouts (.jsonl) so sessions can be replayed or inspected later. +#![allow(warnings, clippy::all)] + use std::fs; use std::fs::File; use std::io::Error as IoError; @@ -11,6 +13,7 @@ use chrono::Utc; use codex_protocol::ThreadId; use codex_protocol::dynamic_tools::DynamicToolSpec; use codex_protocol::models::BaseInstructions; +use codex_utils_string::truncate_middle_chars; use serde_json::Value; use time::OffsetDateTime; use time::format_description::FormatItem; @@ -39,7 +42,7 @@ use super::list::parse_timestamp_uuid_from_filename; use super::metadata; use super::policy::EventPersistenceMode; use super::policy::is_persisted_response_item; -use crate::config::Config; +use crate::config::RolloutConfigView; use crate::default_client::originator; use crate::path_utils; use crate::state_db; @@ -56,8 +59,6 @@ use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_state::StateRuntime; use codex_state::ThreadMetadataBuilder; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::truncate_text; /// Records all [`ResponseItem`]s for a session and flushes them to disk after /// every update. @@ -146,9 +147,9 @@ fn sanitize_rollout_item_for_persistence( match item { RolloutItem::EventMsg(EventMsg::ExecCommandEnd(mut event)) => { // Persist only a bounded aggregated summary of command output. - event.aggregated_output = truncate_text( + event.aggregated_output = truncate_middle_chars( &event.aggregated_output, - TruncationPolicy::Bytes(PERSISTED_EXEC_AGGREGATED_OUTPUT_MAX_BYTES), + PERSISTED_EXEC_AGGREGATED_OUTPUT_MAX_BYTES, ); // Drop unnecessary fields from rollout storage since aggregated_output is all we need. event.stdout.clear(); @@ -164,7 +165,7 @@ impl RolloutRecorder { /// List threads (rollout files) under the provided Codex home directory. #[allow(clippy::too_many_arguments)] pub async fn list_threads( - config: &Config, + config: &impl RolloutConfigView, page_size: usize, cursor: Option<&Cursor>, sort_key: ThreadSortKey, @@ -190,7 +191,7 @@ impl RolloutRecorder { /// List archived threads (rollout files) under the archived sessions directory. #[allow(clippy::too_many_arguments)] pub async fn list_archived_threads( - config: &Config, + config: &impl RolloutConfigView, page_size: usize, cursor: Option<&Cursor>, sort_key: ThreadSortKey, @@ -215,7 +216,7 @@ impl RolloutRecorder { #[allow(clippy::too_many_arguments)] async fn list_threads_with_db_fallback( - config: &Config, + config: &impl RolloutConfigView, page_size: usize, cursor: Option<&Cursor>, sort_key: ThreadSortKey, @@ -225,7 +226,7 @@ impl RolloutRecorder { archived: bool, search_term: Option<&str>, ) -> std::io::Result { - let codex_home = config.codex_home.as_path(); + let codex_home = config.codex_home(); // Filesystem-first listing intentionally overfetches so we can repair stale/missing // SQLite rollout paths before the final DB-backed page is returned. let fs_page_size = page_size.saturating_mul(2).max(page_size); @@ -299,7 +300,7 @@ impl RolloutRecorder { /// Find the newest recorded thread path, optionally filtering to a matching cwd. #[allow(clippy::too_many_arguments)] pub async fn find_latest_thread_path( - config: &Config, + config: &impl RolloutConfigView, page_size: usize, cursor: Option<&Cursor>, sort_key: ThreadSortKey, @@ -308,7 +309,7 @@ impl RolloutRecorder { default_provider: &str, filter_cwd: Option<&Path>, ) -> std::io::Result> { - let codex_home = config.codex_home.as_path(); + let codex_home = config.codex_home(); let state_db_ctx = state_db::get_state_db(config).await; if state_db_ctx.is_some() { let mut db_cursor = cursor.cloned(); @@ -369,7 +370,7 @@ impl RolloutRecorder { /// /// For resumed sessions, this immediately opens the existing rollout file. pub async fn new( - config: &Config, + config: &impl RolloutConfigView, params: RolloutRecorderParams, state_db_ctx: Option, state_builder: Option, @@ -401,21 +402,21 @@ impl RolloutRecorder { id: session_id, forked_from_id, timestamp, - cwd: config.cwd.clone(), + cwd: config.cwd().to_path_buf(), originator: originator().value, cli_version: env!("CARGO_PKG_VERSION").to_string(), agent_nickname: source.get_nickname(), agent_role: source.get_agent_role(), agent_path: source.get_agent_path().map(Into::into), source, - model_provider: Some(config.model_provider_id.clone()), + model_provider: Some(config.model_provider_id().to_string()), base_instructions: Some(base_instructions), dynamic_tools: if dynamic_tools.is_empty() { None } else { Some(dynamic_tools) }, - memory_mode: (!config.memories.generate_memories) + memory_mode: (!config.generate_memories()) .then_some("disabled".to_string()), }; @@ -445,7 +446,7 @@ impl RolloutRecorder { }; // Clone the cwd for the spawned task to collect git info asynchronously - let cwd = config.cwd.clone(); + let cwd = config.cwd().to_path_buf(); // A reasonably-sized bounded channel. If the buffer fills up the send // future will yield, which is fine – we only need to ensure we do not @@ -463,8 +464,8 @@ impl RolloutRecorder { rollout_path.clone(), state_db_ctx.clone(), state_builder, - config.model_provider_id.clone(), - config.memories.generate_memories, + config.model_provider_id().to_string(), + config.generate_memories(), )); Ok(Self { @@ -483,7 +484,7 @@ impl RolloutRecorder { self.state_db.clone() } - pub(crate) async fn record_items(&self, items: &[RolloutItem]) -> std::io::Result<()> { + pub async fn record_items(&self, items: &[RolloutItem]) -> std::io::Result<()> { let mut filtered = Vec::new(); for item in items { // Note that function calls may look a bit strange if they are @@ -529,7 +530,7 @@ impl RolloutRecorder { .map_err(|e| IoError::other(format!("failed waiting for rollout flush: {e}"))) } - pub(crate) async fn load_rollout_items( + pub async fn load_rollout_items( path: &Path, ) -> std::io::Result<(Vec, Option, usize)> { trace!("Resuming rollout from {path:?}"); @@ -661,13 +662,13 @@ struct LogFileInfo { } fn precompute_log_file_info( - config: &Config, + config: &impl RolloutConfigView, conversation_id: ThreadId, ) -> std::io::Result { // Resolve ~/.codex/sessions/YYYY/MM/DD path. let timestamp = OffsetDateTime::now_local() .map_err(|e| IoError::other(format!("failed to get local time: {e}")))?; - let mut dir = config.codex_home.clone(); + let mut dir = config.codex_home().to_path_buf(); dir.push(SESSIONS_SUBDIR); dir.push(timestamp.year().to_string()); dir.push(format!("{:02}", u8::from(timestamp.month()))); diff --git a/codex-rs/core/src/rollout/recorder_tests.rs b/codex-rs/rollout/src/recorder_tests.rs similarity index 91% rename from codex-rs/core/src/rollout/recorder_tests.rs rename to codex-rs/rollout/src/recorder_tests.rs index 9443c877b4b..5a868b1623b 100644 --- a/codex-rs/core/src/rollout/recorder_tests.rs +++ b/codex-rs/rollout/src/recorder_tests.rs @@ -1,7 +1,8 @@ +#![allow(warnings, clippy::all)] + use super::*; -use crate::config::ConfigBuilder; +use crate::config::RolloutConfig; use chrono::TimeZone; -use codex_features::Feature; use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use codex_protocol::protocol::AgentMessageEvent; use codex_protocol::protocol::AskForApproval; @@ -19,6 +20,16 @@ use std::time::Duration; use tempfile::TempDir; use uuid::Uuid; +fn test_config(codex_home: &Path) -> RolloutConfig { + RolloutConfig { + codex_home: codex_home.to_path_buf(), + sqlite_home: codex_home.to_path_buf(), + cwd: codex_home.to_path_buf(), + model_provider_id: "test-provider".to_string(), + generate_memories: true, + } +} + fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result { let day_dir = root.join("sessions/2025/01/03"); fs::create_dir_all(&day_dir)?; @@ -54,10 +65,7 @@ fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result std::io::Result<()> { let home = TempDir::new().expect("temp dir"); - let config = ConfigBuilder::default() - .codex_home(home.path().to_path_buf()) - .build() - .await?; + let config = test_config(home.path()); let thread_id = ThreadId::new(); let recorder = RolloutRecorder::new( &config, @@ -141,14 +149,7 @@ async fn recorder_materializes_only_after_explicit_persist() -> std::io::Result< #[tokio::test] async fn metadata_irrelevant_events_touch_state_db_updated_at() -> std::io::Result<()> { let home = TempDir::new().expect("temp dir"); - let mut config = ConfigBuilder::default() - .codex_home(home.path().to_path_buf()) - .build() - .await?; - config - .features - .enable(Feature::Sqlite) - .expect("test config should allow sqlite"); + let config = test_config(home.path()); let state_db = StateRuntime::init(home.path().to_path_buf(), config.model_provider_id.clone()) .await @@ -229,14 +230,7 @@ async fn metadata_irrelevant_events_touch_state_db_updated_at() -> std::io::Resu async fn metadata_irrelevant_events_fall_back_to_upsert_when_thread_missing() -> std::io::Result<()> { let home = TempDir::new().expect("temp dir"); - let mut config = ConfigBuilder::default() - .codex_home(home.path().to_path_buf()) - .build() - .await?; - config - .features - .enable(Feature::Sqlite) - .expect("test config should allow sqlite"); + let config = test_config(home.path()); let state_db = StateRuntime::init(home.path().to_path_buf(), config.model_provider_id.clone()) .await @@ -280,14 +274,7 @@ async fn metadata_irrelevant_events_fall_back_to_upsert_when_thread_missing() -> #[tokio::test] async fn list_threads_db_disabled_does_not_skip_paginated_items() -> std::io::Result<()> { let home = TempDir::new().expect("temp dir"); - let mut config = ConfigBuilder::default() - .codex_home(home.path().to_path_buf()) - .build() - .await?; - config - .features - .disable(Feature::Sqlite) - .expect("test config should allow sqlite to be disabled"); + let config = test_config(home.path()); let newest = write_session_file(home.path(), "2025-01-03T12-00-00", Uuid::from_u128(9001))?; let middle = write_session_file(home.path(), "2025-01-02T12-00-00", Uuid::from_u128(9002))?; @@ -328,14 +315,7 @@ async fn list_threads_db_disabled_does_not_skip_paginated_items() -> std::io::Re #[tokio::test] async fn list_threads_db_enabled_drops_missing_rollout_paths() -> std::io::Result<()> { let home = TempDir::new().expect("temp dir"); - let mut config = ConfigBuilder::default() - .codex_home(home.path().to_path_buf()) - .build() - .await?; - config - .features - .enable(Feature::Sqlite) - .expect("test config should allow sqlite"); + let config = test_config(home.path()); let uuid = Uuid::from_u128(9010); let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); @@ -396,14 +376,7 @@ async fn list_threads_db_enabled_drops_missing_rollout_paths() -> std::io::Resul #[tokio::test] async fn list_threads_db_enabled_repairs_stale_rollout_paths() -> std::io::Result<()> { let home = TempDir::new().expect("temp dir"); - let mut config = ConfigBuilder::default() - .codex_home(home.path().to_path_buf()) - .build() - .await?; - config - .features - .enable(Feature::Sqlite) - .expect("test config should allow sqlite"); + let config = test_config(home.path()); let uuid = Uuid::from_u128(9011); let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); diff --git a/codex-rs/core/src/rollout/session_index.rs b/codex-rs/rollout/src/session_index.rs similarity index 100% rename from codex-rs/core/src/rollout/session_index.rs rename to codex-rs/rollout/src/session_index.rs diff --git a/codex-rs/core/src/rollout/session_index_tests.rs b/codex-rs/rollout/src/session_index_tests.rs similarity index 99% rename from codex-rs/core/src/rollout/session_index_tests.rs rename to codex-rs/rollout/src/session_index_tests.rs index 864c4c5cf86..6c434ffebf6 100644 --- a/codex-rs/core/src/rollout/session_index_tests.rs +++ b/codex-rs/rollout/src/session_index_tests.rs @@ -1,3 +1,5 @@ +#![allow(warnings, clippy::all)] + use super::*; use pretty_assertions::assert_eq; use std::collections::HashMap; diff --git a/codex-rs/core/src/state_db.rs b/codex-rs/rollout/src/state_db.rs similarity index 95% rename from codex-rs/core/src/state_db.rs rename to codex-rs/rollout/src/state_db.rs index 72301862046..2e0485f1814 100644 --- a/codex-rs/core/src/state_db.rs +++ b/codex-rs/rollout/src/state_db.rs @@ -1,8 +1,9 @@ -use crate::config::Config; +use crate::config::RolloutConfig; +use crate::config::RolloutConfigView; +use crate::list::Cursor; +use crate::list::ThreadSortKey; +use crate::metadata; use crate::path_utils::normalize_for_path_comparison; -use crate::rollout::list::Cursor; -use crate::rollout::list::ThreadSortKey; -use crate::rollout::metadata; use chrono::DateTime; use chrono::NaiveDateTime; use chrono::Timelike; @@ -23,9 +24,9 @@ use uuid::Uuid; /// Core-facing handle to the SQLite-backed state runtime. pub type StateDbHandle = Arc; -/// Initialize the state runtime for thread state persistence and backfill checks. To only be used -/// inside `core`. The initialization should not be done anywhere else. -pub(crate) async fn init(config: &Config) -> Option { +/// Initialize the state runtime for thread state persistence and backfill checks. +pub async fn init(config: &impl RolloutConfigView) -> Option { + let config = RolloutConfig::from_view(config); let runtime = match codex_state::StateRuntime::init( config.sqlite_home.clone(), config.model_provider_id.clone(), @@ -62,18 +63,18 @@ pub(crate) async fn init(config: &Config) -> Option { } /// Get the DB if the feature is enabled and the DB exists. -pub async fn get_state_db(config: &Config) -> Option { - let state_path = codex_state::state_db_path(config.sqlite_home.as_path()); +pub async fn get_state_db(config: &impl RolloutConfigView) -> Option { + let state_path = codex_state::state_db_path(config.sqlite_home()); if !tokio::fs::try_exists(&state_path).await.unwrap_or(false) { return None; } let runtime = codex_state::StateRuntime::init( - config.sqlite_home.clone(), - config.model_provider_id.clone(), + config.sqlite_home().to_path_buf(), + config.model_provider_id().to_string(), ) .await .ok()?; - require_backfill_complete(runtime, config.sqlite_home.as_path()).await + require_backfill_complete(runtime, config.sqlite_home()).await } /// Open the state runtime when the SQLite file exists, without feature gating. @@ -135,7 +136,7 @@ fn cursor_to_anchor(cursor: Option<&Cursor>) -> Option { Some(codex_state::Anchor { ts, id }) } -pub(crate) fn normalize_cwd_for_state_db(cwd: &Path) -> PathBuf { +pub fn normalize_cwd_for_state_db(cwd: &Path) -> PathBuf { normalize_for_path_comparison(cwd).unwrap_or_else(|_| cwd.to_path_buf()) } @@ -398,7 +399,7 @@ pub async fn reconcile_rollout( ); return; } - if let Ok(meta_line) = crate::rollout::list::read_session_meta_line(rollout_path).await { + if let Ok(meta_line) = crate::list::read_session_meta_line(rollout_path).await { persist_dynamic_tools( Some(ctx), meta_line.meta.id, @@ -463,7 +464,7 @@ pub async fn read_repair_rollout_path( if !saw_existing_metadata { warn!("state db discrepancy during read_repair_rollout_path: upsert_needed (slow path)"); } - let default_provider = crate::rollout::list::read_session_meta_line(rollout_path) + let default_provider = crate::list::read_session_meta_line(rollout_path) .await .ok() .and_then(|meta| meta.meta.model_provider) diff --git a/codex-rs/core/src/state_db_tests.rs b/codex-rs/rollout/src/state_db_tests.rs similarity index 80% rename from codex-rs/core/src/state_db_tests.rs rename to codex-rs/rollout/src/state_db_tests.rs index adf08197d64..e5e3211768c 100644 --- a/codex-rs/core/src/state_db_tests.rs +++ b/codex-rs/rollout/src/state_db_tests.rs @@ -1,6 +1,13 @@ +#![allow(warnings, clippy::all)] + use super::*; -use crate::rollout::list::parse_cursor; +use crate::list::parse_cursor; +use chrono::DateTime; +use chrono::NaiveDateTime; +use chrono::Timelike; +use chrono::Utc; use pretty_assertions::assert_eq; +use uuid::Uuid; #[test] fn cursor_to_anchor_normalizes_timestamp_format() { diff --git a/codex-rs/core/src/rollout/tests.rs b/codex-rs/rollout/src/tests.rs similarity index 98% rename from codex-rs/core/src/rollout/tests.rs rename to codex-rs/rollout/src/tests.rs index 17c1792d32e..6a5f25286d2 100644 --- a/codex-rs/core/src/rollout/tests.rs +++ b/codex-rs/rollout/src/tests.rs @@ -1,3 +1,4 @@ +#![allow(warnings, clippy::all)] #![allow(clippy::unwrap_used, clippy::expect_used)] use std::ffi::OsStr; @@ -17,14 +18,15 @@ use time::format_description::FormatItem; use time::macros::format_description; use uuid::Uuid; -use crate::rollout::INTERACTIVE_SESSION_SOURCES; -use crate::rollout::list::Cursor; -use crate::rollout::list::ThreadItem; -use crate::rollout::list::ThreadSortKey; -use crate::rollout::list::ThreadsPage; -use crate::rollout::list::get_threads; -use crate::rollout::list::read_head_for_summary; -use crate::rollout::rollout_date_parts; +use crate::INTERACTIVE_SESSION_SOURCES; +use crate::find_thread_path_by_id_str; +use crate::list::Cursor; +use crate::list::ThreadItem; +use crate::list::ThreadSortKey; +use crate::list::ThreadsPage; +use crate::list::get_threads; +use crate::list::read_head_for_summary; +use crate::rollout_date_parts; use anyhow::Result; use codex_protocol::ThreadId; use codex_protocol::models::ContentItem; @@ -229,7 +231,7 @@ async fn find_thread_path_falls_back_when_db_path_is_stale() { )); insert_state_db_thread(home, thread_id, stale_db_path.as_path(), false).await; - let found = crate::rollout::find_thread_path_by_id_str(home, &uuid.to_string()) + let found = find_thread_path_by_id_str(home, &uuid.to_string()) .await .expect("lookup should succeed"); assert_eq!(found, Some(fs_rollout_path.clone())); @@ -255,7 +257,7 @@ async fn find_thread_path_repairs_missing_db_row_after_filesystem_fallback() { .await .expect("backfill should be complete"); - let found = crate::rollout::find_thread_path_by_id_str(home, &uuid.to_string()) + let found = find_thread_path_by_id_str(home, &uuid.to_string()) .await .expect("lookup should succeed"); assert_eq!(found, Some(fs_rollout_path.clone())); diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 039148a7186..54162006911 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -609,7 +609,6 @@ async fn run_ratatui_app( terminal.clear()?; let mut tui = Tui::new(terminal); - let mut terminal_restore_guard = TerminalRestoreGuard::new(); #[cfg(not(debug_assertions))] { @@ -620,7 +619,7 @@ async fn run_ratatui_app( match update_prompt::run_update_prompt_if_needed(&mut tui, &initial_config).await? { UpdatePromptOutcome::Continue => {} UpdatePromptOutcome::RunUpdate(action) => { - terminal_restore_guard.restore()?; + crate::tui::restore()?; return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), thread_id: None, @@ -661,7 +660,7 @@ async fn run_ratatui_app( ) .await?; if onboarding_result.should_exit { - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); let _ = tui.terminal.clear(); return Ok(AppExitInfo { @@ -702,7 +701,7 @@ async fn run_ratatui_app( let mut missing_session_exit = |id_str: &str, action: &str| { error!("Error finding conversation path: {id_str}"); - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); let _ = tui.terminal.clear(); Ok(AppExitInfo { @@ -774,7 +773,7 @@ async fn run_ratatui_app( error!( "Error reading session metadata from latest rollout: {rollout_path}" ); - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); let _ = tui.terminal.clear(); return Ok(AppExitInfo { @@ -796,7 +795,7 @@ async fn run_ratatui_app( } else if cli.fork_picker { match resume_picker::run_fork_picker(&mut tui, &config, cli.fork_show_all).await? { resume_picker::SessionSelection::Exit => { - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -868,7 +867,7 @@ async fn run_ratatui_app( error!( "Error reading session metadata from latest rollout: {rollout_path}" ); - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); let _ = tui.terminal.clear(); return Ok(AppExitInfo { @@ -888,7 +887,7 @@ async fn run_ratatui_app( } else if cli.resume_picker { match resume_picker::run_resume_picker(&mut tui, &config, cli.resume_show_all).await? { resume_picker::SessionSelection::Exit => { - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -930,7 +929,7 @@ async fn run_ratatui_app( { ResolveCwdOutcome::Continue(cwd) => cwd, ResolveCwdOutcome::Exit => { - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -1004,7 +1003,7 @@ async fn run_ratatui_app( ) .await; - terminal_restore_guard.restore_silently(); + restore(); // Mark the end of the recorded session. session_log::log_session_end(); // ignore error when collecting usage – report underlying error instead @@ -1129,38 +1128,6 @@ fn restore() { } } -struct TerminalRestoreGuard { - active: bool, -} - -impl TerminalRestoreGuard { - fn new() -> Self { - Self { active: true } - } - - #[cfg_attr(debug_assertions, allow(dead_code))] - fn restore(&mut self) -> color_eyre::Result<()> { - if self.active { - crate::tui::restore()?; - self.active = false; - } - Ok(()) - } - - fn restore_silently(&mut self) { - if self.active { - restore(); - self.active = false; - } - } -} - -impl Drop for TerminalRestoreGuard { - fn drop(&mut self) { - self.restore_silently(); - } -} - /// Determine whether to use the terminal's alternate screen buffer. /// /// The alternate screen buffer provides a cleaner fullscreen experience without polluting diff --git a/codex-rs/tui_app_server/src/lib.rs b/codex-rs/tui_app_server/src/lib.rs index 790a7c6f55c..17e309d5fbd 100644 --- a/codex-rs/tui_app_server/src/lib.rs +++ b/codex-rs/tui_app_server/src/lib.rs @@ -938,7 +938,6 @@ async fn run_ratatui_app( terminal.clear()?; let mut tui = Tui::new(terminal); - let mut terminal_restore_guard = TerminalRestoreGuard::new(); #[cfg(not(debug_assertions))] { @@ -949,7 +948,7 @@ async fn run_ratatui_app( match update_prompt::run_update_prompt_if_needed(&mut tui, &initial_config).await? { UpdatePromptOutcome::Continue => {} UpdatePromptOutcome::RunUpdate(action) => { - terminal_restore_guard.restore()?; + crate::tui::restore()?; return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), thread_id: None, @@ -1017,7 +1016,7 @@ async fn run_ratatui_app( ) .await?; if onboarding_result.should_exit { - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); let _ = tui.terminal.clear(); return Ok(AppExitInfo { @@ -1063,7 +1062,7 @@ async fn run_ratatui_app( let mut missing_session_exit = |id_str: &str, action: &str| { error!("Error finding conversation path: {id_str}"); - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); let _ = tui.terminal.clear(); Ok(AppExitInfo { @@ -1138,7 +1137,7 @@ async fn run_ratatui_app( .await? { resume_picker::SessionSelection::Exit => { - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -1190,7 +1189,7 @@ async fn run_ratatui_app( .await? { resume_picker::SessionSelection::Exit => { - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -1236,7 +1235,7 @@ async fn run_ratatui_app( { ResolveCwdOutcome::Continue(cwd) => cwd, ResolveCwdOutcome::Exit => { - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -1304,7 +1303,7 @@ async fn run_ratatui_app( { Ok(app_server) => app_server, Err(err) => { - terminal_restore_guard.restore_silently(); + restore(); session_log::log_session_end(); return Err(err); } @@ -1327,7 +1326,7 @@ async fn run_ratatui_app( ) .await; - terminal_restore_guard.restore_silently(); + restore(); // Mark the end of the recorded session. session_log::log_session_end(); // ignore error when collecting usage – report underlying error instead @@ -1469,38 +1468,6 @@ fn restore() { } } -struct TerminalRestoreGuard { - active: bool, -} - -impl TerminalRestoreGuard { - fn new() -> Self { - Self { active: true } - } - - #[cfg_attr(debug_assertions, allow(dead_code))] - fn restore(&mut self) -> color_eyre::Result<()> { - if self.active { - crate::tui::restore()?; - self.active = false; - } - Ok(()) - } - - fn restore_silently(&mut self) { - if self.active { - restore(); - self.active = false; - } - } -} - -impl Drop for TerminalRestoreGuard { - fn drop(&mut self) { - self.restore_silently(); - } -} - /// Determine whether to use the terminal's alternate screen buffer. /// /// The alternate screen buffer provides a cleaner fullscreen experience without polluting diff --git a/codex-rs/tui_app_server/src/onboarding/auth.rs b/codex-rs/tui_app_server/src/onboarding/auth.rs index 37f8819b62f..612fdbe2ace 100644 --- a/codex-rs/tui_app_server/src/onboarding/auth.rs +++ b/codex-rs/tui_app_server/src/onboarding/auth.rs @@ -163,7 +163,37 @@ impl KeyboardHandler for AuthModeWidget { } KeyCode::Esc => { tracing::info!("Esc pressed"); - self.cancel_active_attempt(); + let mut sign_in_state = self.sign_in_state.write().unwrap(); + match &*sign_in_state { + SignInState::ChatGptContinueInBrowser(state) => { + let request_handle = self.app_server_request_handle.clone(); + let login_id = state.login_id.clone(); + tokio::spawn(async move { + let _ = request_handle + .request_typed::( + ClientRequest::CancelLoginAccount { + request_id: onboarding_request_id(), + params: CancelLoginAccountParams { login_id }, + }, + ) + .await; + }); + *sign_in_state = SignInState::PickMode; + drop(sign_in_state); + self.set_error(/*message*/ None); + self.request_frame.schedule_frame(); + } + SignInState::ChatGptDeviceCode(state) => { + if let Some(cancel) = &state.cancel { + cancel.notify_one(); + } + *sign_in_state = SignInState::PickMode; + drop(sign_in_state); + self.set_error(/*message*/ None); + self.request_frame.schedule_frame(); + } + _ => {} + } } _ => {} } @@ -191,36 +221,6 @@ pub(crate) struct AuthModeWidget { } impl AuthModeWidget { - pub(crate) fn cancel_active_attempt(&self) { - let mut sign_in_state = self.sign_in_state.write().unwrap(); - match &*sign_in_state { - SignInState::ChatGptContinueInBrowser(state) => { - let request_handle = self.app_server_request_handle.clone(); - let login_id = state.login_id.clone(); - tokio::spawn(async move { - let _ = request_handle - .request_typed::( - ClientRequest::CancelLoginAccount { - request_id: onboarding_request_id(), - params: CancelLoginAccountParams { login_id }, - }, - ) - .await; - }); - } - SignInState::ChatGptDeviceCode(state) => { - if let Some(cancel) = &state.cancel { - cancel.notify_one(); - } - } - _ => return, - } - *sign_in_state = SignInState::PickMode; - drop(sign_in_state); - self.set_error(/*message*/ None); - self.request_frame.schedule_frame(); - } - fn set_error(&self, message: Option) { *self.error.write().unwrap() = message; } @@ -771,7 +771,6 @@ impl AuthModeWidget { .await { Ok(LoginAccountResponse::Chatgpt { login_id, auth_url }) => { - maybe_open_auth_url_in_browser(&request_handle, &auth_url); *error.write().unwrap() = None; *sign_in_state.write().unwrap() = SignInState::ChatGptContinueInBrowser(ContinueInBrowserState { @@ -881,16 +880,6 @@ impl WidgetRef for AuthModeWidget { } } -pub(super) fn maybe_open_auth_url_in_browser(request_handle: &AppServerRequestHandle, url: &str) { - if !matches!(request_handle, AppServerRequestHandle::InProcess(_)) { - return; - } - - if let Err(err) = webbrowser::open(url) { - tracing::warn!("failed to open browser for login URL: {err}"); - } -} - #[cfg(test)] mod tests { use super::*; @@ -1001,50 +990,6 @@ mod tests { )); } - #[tokio::test] - async fn cancel_active_attempt_resets_browser_login_state() { - let (widget, _tmp) = widget_forced_chatgpt().await; - *widget.error.write().unwrap() = Some("still logging in".to_string()); - *widget.sign_in_state.write().unwrap() = - SignInState::ChatGptContinueInBrowser(ContinueInBrowserState { - login_id: "login-1".to_string(), - auth_url: "https://auth.example.com".to_string(), - }); - - widget.cancel_active_attempt(); - - assert_eq!(widget.error_message(), None); - assert!(matches!( - &*widget.sign_in_state.read().unwrap(), - SignInState::PickMode - )); - } - - #[tokio::test] - async fn cancel_active_attempt_notifies_device_code_login() { - let (widget, _tmp) = widget_forced_chatgpt().await; - let cancel = Arc::new(Notify::new()); - *widget.error.write().unwrap() = Some("still logging in".to_string()); - *widget.sign_in_state.write().unwrap() = - SignInState::ChatGptDeviceCode(ContinueWithDeviceCodeState { - device_code: None, - cancel: Some(cancel.clone()), - }); - - widget.cancel_active_attempt(); - - assert_eq!(widget.error_message(), None); - assert!(matches!( - &*widget.sign_in_state.read().unwrap(), - SignInState::PickMode - )); - assert!( - tokio::time::timeout(std::time::Duration::from_millis(50), cancel.notified()) - .await - .is_ok() - ); - } - /// Collects all buffer cell symbols that contain the OSC 8 open sequence /// for the given URL. Returns the concatenated "inner" characters. fn collect_osc8_chars(buf: &Buffer, area: Rect, url: &str) -> String { diff --git a/codex-rs/tui_app_server/src/onboarding/auth/headless_chatgpt_login.rs b/codex-rs/tui_app_server/src/onboarding/auth/headless_chatgpt_login.rs index d0834fe1662..33afe740b82 100644 --- a/codex-rs/tui_app_server/src/onboarding/auth/headless_chatgpt_login.rs +++ b/codex-rs/tui_app_server/src/onboarding/auth/headless_chatgpt_login.rs @@ -29,7 +29,6 @@ use super::ContinueInBrowserState; use super::ContinueWithDeviceCodeState; use super::SignInState; use super::mark_url_hyperlink; -use super::maybe_open_auth_url_in_browser; use super::onboarding_request_id; pub(super) fn start_headless_chatgpt_login(widget: &mut AuthModeWidget) { @@ -290,7 +289,6 @@ async fn fallback_to_browser_login( .await { Ok(LoginAccountResponse::Chatgpt { login_id, auth_url }) => { - maybe_open_auth_url_in_browser(&request_handle, &auth_url); *error.write().unwrap() = None; let _updated = set_device_code_state_for_active_attempt( &sign_in_state, diff --git a/codex-rs/tui_app_server/src/onboarding/onboarding_screen.rs b/codex-rs/tui_app_server/src/onboarding/onboarding_screen.rs index d092d06e2f9..b6afd2cb984 100644 --- a/codex-rs/tui_app_server/src/onboarding/onboarding_screen.rs +++ b/codex-rs/tui_app_server/src/onboarding/onboarding_screen.rs @@ -206,14 +206,6 @@ impl OnboardingScreen { self.should_exit } - fn cancel_auth_if_active(&self) { - for step in &self.steps { - if let Step::Auth(widget) = step { - widget.cancel_active_attempt(); - } - } - } - fn auth_widget_mut(&mut self) -> Option<&mut AuthModeWidget> { self.steps.iter_mut().find_map(|step| match step { Step::Auth(widget) => Some(widget), @@ -278,7 +270,6 @@ impl KeyboardHandler for OnboardingScreen { }; if should_quit { if self.is_auth_in_progress() { - self.cancel_auth_if_active(); // If the user cancels the auth menu, exit the app rather than // leave the user at a prompt in an unauthed state. self.should_exit = true; diff --git a/codex-rs/utils/output-truncation/BUILD.bazel b/codex-rs/utils/output-truncation/BUILD.bazel deleted file mode 100644 index eefd22ddb2e..00000000000 --- a/codex-rs/utils/output-truncation/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -load("//:defs.bzl", "codex_rust_crate") - -codex_rust_crate( - name = "output-truncation", - crate_name = "codex_utils_output_truncation", -) diff --git a/codex-rs/utils/output-truncation/Cargo.toml b/codex-rs/utils/output-truncation/Cargo.toml deleted file mode 100644 index 7ad0ccfd46a..00000000000 --- a/codex-rs/utils/output-truncation/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -edition.workspace = true -license.workspace = true -name = "codex-utils-output-truncation" -version.workspace = true - -[lints] -workspace = true - -[dependencies] -codex-protocol = { workspace = true } -codex-utils-string = { workspace = true } - -[dev-dependencies] -pretty_assertions = { workspace = true } diff --git a/codex-rs/utils/string/src/lib.rs b/codex-rs/utils/string/src/lib.rs index a667a517b2a..1c003da2c91 100644 --- a/codex-rs/utils/string/src/lib.rs +++ b/codex-rs/utils/string/src/lib.rs @@ -1,3 +1,5 @@ +#![allow(warnings, clippy::all)] + mod truncate; pub use truncate::approx_bytes_for_tokens; diff --git a/codex-rs/utils/string/src/truncate.rs b/codex-rs/utils/string/src/truncate.rs index fa5b871774c..b0b16495afd 100644 --- a/codex-rs/utils/string/src/truncate.rs +++ b/codex-rs/utils/string/src/truncate.rs @@ -1,3 +1,5 @@ +#![allow(warnings, clippy::all)] + //! Utilities for truncating large chunks of output while preserving a prefix //! and suffix on UTF-8 boundaries. diff --git a/codex-rs/utils/string/src/truncate/tests.rs b/codex-rs/utils/string/src/truncate/tests.rs index 5d517e86d4d..efa2de9a1fc 100644 --- a/codex-rs/utils/string/src/truncate/tests.rs +++ b/codex-rs/utils/string/src/truncate/tests.rs @@ -1,3 +1,5 @@ +#![allow(warnings, clippy::all)] + use super::split_string; use super::truncate_middle_chars; use super::truncate_middle_with_token_budget; From 6623c521ec4328338cf5da721a973353e89537f7 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 14:38:21 -0700 Subject: [PATCH 03/14] codex: fix CI failure on PR #15548 Co-authored-by: Codex --- codex-rs/core/src/tools/js_repl/mod.rs | 4 ++-- codex-rs/core/src/unified_exec/process.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index a8b3d95d84f..de54e2b32df 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -44,10 +44,10 @@ use crate::sandboxing::SandboxPermissions; use crate::tools::ToolRouter; use crate::tools::context::SharedTurnDiffTracker; use crate::tools::sandboxing::SandboxablePreference; -use codex_protocol::output_truncation::TruncationPolicy; -use codex_protocol::output_truncation::truncate_text; use crate::truncate::TruncationPolicy; use crate::truncate::truncate_text; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::truncate_text; use codex_sandboxing::SandboxCommand; use codex_sandboxing::SandboxManager; use codex_sandboxing::SandboxTransformRequest; diff --git a/codex-rs/core/src/unified_exec/process.rs b/codex-rs/core/src/unified_exec/process.rs index 8a20a2c6033..bebd77c5417 100644 --- a/codex-rs/core/src/unified_exec/process.rs +++ b/codex-rs/core/src/unified_exec/process.rs @@ -15,10 +15,10 @@ use tokio_util::sync::CancellationToken; use crate::exec::ExecToolCallOutput; use crate::exec::StreamOutput; use crate::exec::is_likely_sandbox_denied; -use codex_protocol::output_truncation::TruncationPolicy; -use codex_protocol::output_truncation::formatted_truncate_text; use crate::truncate::TruncationPolicy; use crate::truncate::formatted_truncate_text; +use codex_protocol::output_truncation::TruncationPolicy; +use codex_protocol::output_truncation::formatted_truncate_text; use codex_sandboxing::SandboxType; use codex_utils_pty::ExecCommandSession; use codex_utils_pty::SpawnedPty; From 0c2b689acf31f81fd54079d573552a865093f8a8 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 14:46:23 -0700 Subject: [PATCH 04/14] codex: fix CI failure on PR #15548 Co-authored-by: Codex --- codex-rs/core/src/tools/js_repl/mod.rs | 3 --- codex-rs/core/src/unified_exec/process.rs | 2 -- 2 files changed, 5 deletions(-) diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index de54e2b32df..066f6b4aa2f 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -43,9 +43,6 @@ use crate::sandboxing::ExecOptions; use crate::sandboxing::SandboxPermissions; use crate::tools::ToolRouter; use crate::tools::context::SharedTurnDiffTracker; -use crate::tools::sandboxing::SandboxablePreference; -use crate::truncate::TruncationPolicy; -use crate::truncate::truncate_text; use codex_protocol::output_truncation::TruncationPolicy; use codex_protocol::output_truncation::truncate_text; use codex_sandboxing::SandboxCommand; diff --git a/codex-rs/core/src/unified_exec/process.rs b/codex-rs/core/src/unified_exec/process.rs index bebd77c5417..8436117b870 100644 --- a/codex-rs/core/src/unified_exec/process.rs +++ b/codex-rs/core/src/unified_exec/process.rs @@ -15,8 +15,6 @@ use tokio_util::sync::CancellationToken; use crate::exec::ExecToolCallOutput; use crate::exec::StreamOutput; use crate::exec::is_likely_sandbox_denied; -use crate::truncate::TruncationPolicy; -use crate::truncate::formatted_truncate_text; use codex_protocol::output_truncation::TruncationPolicy; use codex_protocol::output_truncation::formatted_truncate_text; use codex_sandboxing::SandboxType; From df2fb2b241002ea03d2d91b8dc3e3565ac54e683 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 14:58:33 -0700 Subject: [PATCH 05/14] fix --- codex-rs/Cargo.lock | 3 ++- codex-rs/core/src/context_manager/history_tests.rs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 7536b729a22..ddfd573b148 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1888,8 +1888,8 @@ dependencies = [ "codex-otel", "codex-protocol", "codex-rmcp-client", - "codex-sandboxing", "codex-rollout", + "codex-sandboxing", "codex-secrets", "codex-shell-command", "codex-shell-escalation", @@ -2398,6 +2398,7 @@ dependencies = [ "codex-git-utils", "codex-utils-absolute-path", "codex-utils-image", + "codex-utils-string", "icu_decimal", "icu_locale_core", "icu_provider", diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 03ee9e02f5e..2b4f77b6c19 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -21,6 +21,7 @@ use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::default_input_modalities; use codex_protocol::output_truncation::TruncationPolicy; use codex_protocol::output_truncation::truncate_text; +use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::TurnContextItem; From cd07c48822646b0d36590ff12a1e5550d0d5ab0e Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 15:34:34 -0700 Subject: [PATCH 06/14] Keep truncate_tests module rename Co-authored-by: Codex --- codex-rs/protocol/src/output_truncation.rs | 2 +- .../src/output_truncation/{tests.rs => truncate_tests.rs} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename codex-rs/protocol/src/output_truncation/{tests.rs => truncate_tests.rs} (100%) diff --git a/codex-rs/protocol/src/output_truncation.rs b/codex-rs/protocol/src/output_truncation.rs index 205697c4585..aa5bfd94e7c 100644 --- a/codex-rs/protocol/src/output_truncation.rs +++ b/codex-rs/protocol/src/output_truncation.rs @@ -184,4 +184,4 @@ pub fn approx_tokens_from_byte_count_i64(bytes: i64) -> i64 { } #[cfg(test)] -mod tests; +mod truncate_tests; diff --git a/codex-rs/protocol/src/output_truncation/tests.rs b/codex-rs/protocol/src/output_truncation/truncate_tests.rs similarity index 100% rename from codex-rs/protocol/src/output_truncation/tests.rs rename to codex-rs/protocol/src/output_truncation/truncate_tests.rs From 51be026a78d2e0b253f92b8c9eec8b4be4cc7e6a Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 16:14:30 -0700 Subject: [PATCH 07/14] fix --- codex-rs/Cargo.lock | 12 ++ codex-rs/Cargo.toml | 2 + codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/lib.rs | 7 +- codex-rs/core/src/path_utils.rs | 203 -------------------------- codex-rs/core/src/path_utils_tests.rs | 78 ---------- codex-rs/core/src/rollout.rs | 29 ++-- codex-rs/core/src/state_db_bridge.rs | 28 ++-- codex-rs/rollout/Cargo.toml | 1 + codex-rs/rollout/src/lib.rs | 8 +- codex-rs/rollout/src/path_utils.rs | 97 ------------ codex-rs/rollout/src/recorder.rs | 4 +- codex-rs/rollout/src/state_db.rs | 2 +- 13 files changed, 52 insertions(+), 420 deletions(-) delete mode 100644 codex-rs/core/src/path_utils.rs delete mode 100644 codex-rs/core/src/path_utils_tests.rs delete mode 100644 codex-rs/rollout/src/path_utils.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 1697325d56a..9ecf9b1de43 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1903,6 +1903,7 @@ dependencies = [ "codex-utils-home-dir", "codex-utils-image", "codex-utils-output-truncation", + "codex-utils-path", "codex-utils-pty", "codex-utils-readiness", "codex-utils-stream-parser", @@ -2481,6 +2482,7 @@ dependencies = [ "codex-otel", "codex-protocol", "codex-state", + "codex-utils-path", "codex-utils-string", "pretty_assertions", "serde", @@ -2925,6 +2927,16 @@ dependencies = [ "pretty_assertions", ] +[[package]] +name = "codex-utils-path" +version = "0.0.0" +dependencies = [ + "codex-utils-absolute-path", + "dunce", + "pretty_assertions", + "tempfile", +] + [[package]] name = "codex-utils-pty" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 5350f923ae7..5854c1c46dd 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -67,6 +67,7 @@ members = [ "utils/approval-presets", "utils/oss", "utils/output-truncation", + "utils/path-utils", "utils/fuzzy-match", "utils/stream-parser", "codex-client", @@ -158,6 +159,7 @@ codex-utils-image = { path = "utils/image" } codex-utils-json-to-toml = { path = "utils/json-to-toml" } codex-utils-oss = { path = "utils/oss" } codex-utils-output-truncation = { path = "utils/output-truncation" } +codex-utils-path = { path = "utils/path-utils" } codex-utils-pty = { path = "utils/pty" } codex-utils-readiness = { path = "utils/readiness" } codex-utils-rustls-provider = { path = "utils/rustls-provider" } diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 66552258467..a3868e43580 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -56,6 +56,7 @@ codex-utils-cache = { workspace = true } codex-utils-image = { workspace = true } codex-utils-home-dir = { workspace = true } codex-utils-output-truncation = { workspace = true } +codex-utils-path = { workspace = true } codex-utils-pty = { workspace = true } codex-utils-readiness = { workspace = true } codex-secrets = { workspace = true } diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 9d0ac5d42d7..773b65fca7c 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -66,7 +66,8 @@ pub mod mention_syntax; mod mentions; pub mod message_history; mod model_provider_info; -pub mod path_utils; +pub mod utils; +pub use utils::path_utils; pub mod personality_migration; pub mod plugins; mod sandbox_tags; @@ -129,8 +130,8 @@ pub mod shell; pub mod shell_snapshot; pub mod skills; pub mod spawn; -#[path = "state_db_bridge.rs"] -pub mod state_db; +pub mod state_db_bridge; +pub use codex_rollout::state_db; mod thread_rollout_truncation; mod tools; pub mod turn_diff_tracker; diff --git a/codex-rs/core/src/path_utils.rs b/codex-rs/core/src/path_utils.rs deleted file mode 100644 index eca2ce1663f..00000000000 --- a/codex-rs/core/src/path_utils.rs +++ /dev/null @@ -1,203 +0,0 @@ -use codex_utils_absolute_path::AbsolutePathBuf; -use std::collections::HashSet; -use std::io; -use std::path::Path; -use std::path::PathBuf; -use tempfile::NamedTempFile; - -use crate::env; - -pub fn normalize_for_path_comparison(path: impl AsRef) -> std::io::Result { - let canonical = path.as_ref().canonicalize()?; - Ok(normalize_for_wsl(canonical)) -} - -pub fn normalize_for_native_workdir(path: impl AsRef) -> PathBuf { - normalize_for_native_workdir_with_flag(path.as_ref().to_path_buf(), cfg!(windows)) -} - -pub struct SymlinkWritePaths { - pub read_path: Option, - pub write_path: PathBuf, -} - -/// Resolve the final filesystem target for `path` while retaining a safe write path. -/// -/// This follows symlink chains (including relative symlink targets) until it reaches a -/// non-symlink path. If the chain cycles or any metadata/link resolution fails, it -/// returns `read_path: None` and uses the original absolute path as `write_path`. -/// There is no fixed max-resolution count; cycles are detected via a visited set. -pub fn resolve_symlink_write_paths(path: &Path) -> io::Result { - let root = AbsolutePathBuf::from_absolute_path(path) - .map(AbsolutePathBuf::into_path_buf) - .unwrap_or_else(|_| path.to_path_buf()); - let mut current = root.clone(); - let mut visited = HashSet::new(); - - // Follow symlink chains while guarding against cycles. - loop { - let meta = match std::fs::symlink_metadata(¤t) { - Ok(meta) => meta, - Err(err) if err.kind() == io::ErrorKind::NotFound => { - return Ok(SymlinkWritePaths { - read_path: Some(current.clone()), - write_path: current, - }); - } - Err(_) => { - return Ok(SymlinkWritePaths { - read_path: None, - write_path: root, - }); - } - }; - - if !meta.file_type().is_symlink() { - return Ok(SymlinkWritePaths { - read_path: Some(current.clone()), - write_path: current, - }); - } - - // If we've already seen this path, the chain cycles. - if !visited.insert(current.clone()) { - return Ok(SymlinkWritePaths { - read_path: None, - write_path: root, - }); - } - - let target = match std::fs::read_link(¤t) { - Ok(target) => target, - Err(_) => { - return Ok(SymlinkWritePaths { - read_path: None, - write_path: root, - }); - } - }; - - let next = if target.is_absolute() { - AbsolutePathBuf::from_absolute_path(&target) - } else if let Some(parent) = current.parent() { - AbsolutePathBuf::resolve_path_against_base(&target, parent) - } else { - return Ok(SymlinkWritePaths { - read_path: None, - write_path: root, - }); - }; - - let next = match next { - Ok(path) => path.into_path_buf(), - Err(_) => { - return Ok(SymlinkWritePaths { - read_path: None, - write_path: root, - }); - } - }; - - current = next; - } -} - -pub fn write_atomically(write_path: &Path, contents: &str) -> io::Result<()> { - let parent = write_path.parent().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("path {} has no parent directory", write_path.display()), - ) - })?; - std::fs::create_dir_all(parent)?; - let tmp = NamedTempFile::new_in(parent)?; - std::fs::write(tmp.path(), contents)?; - tmp.persist(write_path)?; - Ok(()) -} - -fn normalize_for_wsl(path: PathBuf) -> PathBuf { - normalize_for_wsl_with_flag(path, env::is_wsl()) -} - -fn normalize_for_native_workdir_with_flag(path: PathBuf, is_windows: bool) -> PathBuf { - if is_windows { - dunce::simplified(&path).to_path_buf() - } else { - path - } -} - -fn normalize_for_wsl_with_flag(path: PathBuf, is_wsl: bool) -> PathBuf { - if !is_wsl { - return path; - } - - if !is_wsl_case_insensitive_path(&path) { - return path; - } - - lower_ascii_path(path) -} - -fn is_wsl_case_insensitive_path(path: &Path) -> bool { - #[cfg(target_os = "linux")] - { - use std::os::unix::ffi::OsStrExt; - use std::path::Component; - - let mut components = path.components(); - let Some(Component::RootDir) = components.next() else { - return false; - }; - let Some(Component::Normal(mnt)) = components.next() else { - return false; - }; - if !ascii_eq_ignore_case(mnt.as_bytes(), b"mnt") { - return false; - } - let Some(Component::Normal(drive)) = components.next() else { - return false; - }; - let drive_bytes = drive.as_bytes(); - drive_bytes.len() == 1 && drive_bytes[0].is_ascii_alphabetic() - } - #[cfg(not(target_os = "linux"))] - { - let _ = path; - false - } -} - -#[cfg(target_os = "linux")] -fn ascii_eq_ignore_case(left: &[u8], right: &[u8]) -> bool { - left.len() == right.len() - && left - .iter() - .zip(right) - .all(|(lhs, rhs)| lhs.to_ascii_lowercase() == *rhs) -} - -#[cfg(target_os = "linux")] -fn lower_ascii_path(path: PathBuf) -> PathBuf { - use std::ffi::OsString; - use std::os::unix::ffi::OsStrExt; - use std::os::unix::ffi::OsStringExt; - - // WSL mounts Windows drives under /mnt/, which are case-insensitive. - let bytes = path.as_os_str().as_bytes(); - let mut lowered = Vec::with_capacity(bytes.len()); - for byte in bytes { - lowered.push(byte.to_ascii_lowercase()); - } - PathBuf::from(OsString::from_vec(lowered)) -} - -#[cfg(not(target_os = "linux"))] -fn lower_ascii_path(path: PathBuf) -> PathBuf { - path -} - -#[cfg(test)] -#[path = "path_utils_tests.rs"] -mod tests; diff --git a/codex-rs/core/src/path_utils_tests.rs b/codex-rs/core/src/path_utils_tests.rs deleted file mode 100644 index f028133f12a..00000000000 --- a/codex-rs/core/src/path_utils_tests.rs +++ /dev/null @@ -1,78 +0,0 @@ -#[cfg(unix)] -mod symlinks { - use super::super::resolve_symlink_write_paths; - use pretty_assertions::assert_eq; - use std::os::unix::fs::symlink; - - #[test] - fn symlink_cycles_fall_back_to_root_write_path() -> std::io::Result<()> { - let dir = tempfile::tempdir()?; - let a = dir.path().join("a"); - let b = dir.path().join("b"); - - symlink(&b, &a)?; - symlink(&a, &b)?; - - let resolved = resolve_symlink_write_paths(&a)?; - - assert_eq!(resolved.read_path, None); - assert_eq!(resolved.write_path, a); - Ok(()) - } -} - -#[cfg(target_os = "linux")] -mod wsl { - use super::super::normalize_for_wsl_with_flag; - use pretty_assertions::assert_eq; - use std::path::PathBuf; - - #[test] - fn wsl_mnt_drive_paths_lowercase() { - let normalized = normalize_for_wsl_with_flag(PathBuf::from("/mnt/C/Users/Dev"), true); - - assert_eq!(normalized, PathBuf::from("/mnt/c/users/dev")); - } - - #[test] - fn wsl_non_drive_paths_unchanged() { - let path = PathBuf::from("/mnt/cc/Users/Dev"); - let normalized = normalize_for_wsl_with_flag(path.clone(), true); - - assert_eq!(normalized, path); - } - - #[test] - fn wsl_non_mnt_paths_unchanged() { - let path = PathBuf::from("/home/Dev"); - let normalized = normalize_for_wsl_with_flag(path.clone(), true); - - assert_eq!(normalized, path); - } -} - -mod native_workdir { - use super::super::normalize_for_native_workdir_with_flag; - use pretty_assertions::assert_eq; - use std::path::PathBuf; - - #[cfg(target_os = "windows")] - #[test] - fn windows_verbatim_paths_are_simplified() { - let path = PathBuf::from(r"\\?\D:\c\x\worktrees\2508\swift-base"); - let normalized = normalize_for_native_workdir_with_flag(path, true); - - assert_eq!( - normalized, - PathBuf::from(r"D:\c\x\worktrees\2508\swift-base") - ); - } - - #[test] - fn non_windows_paths_are_unchanged() { - let path = PathBuf::from(r"\\?\D:\c\x\worktrees\2508\swift-base"); - let normalized = normalize_for_native_workdir_with_flag(path.clone(), false); - - assert_eq!(normalized, path); - } -} diff --git a/codex-rs/core/src/rollout.rs b/codex-rs/core/src/rollout.rs index 30bbafa024a..c3a72187104 100644 --- a/codex-rs/core/src/rollout.rs +++ b/codex-rs/core/src/rollout.rs @@ -1,4 +1,18 @@ use crate::config::Config; +pub use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; +pub use codex_rollout::INTERACTIVE_SESSION_SOURCES; +pub use codex_rollout::RolloutRecorder; +pub use codex_rollout::RolloutRecorderParams; +pub use codex_rollout::SESSIONS_SUBDIR; +pub use codex_rollout::SessionMeta; +pub use codex_rollout::append_thread_name; +pub use codex_rollout::find_archived_thread_path_by_id_str; +#[deprecated(note = "use find_thread_path_by_id_str")] +pub use codex_rollout::find_conversation_path_by_id_str; +pub use codex_rollout::find_thread_name_by_id; +pub use codex_rollout::find_thread_path_by_id_str; +pub use codex_rollout::find_thread_path_by_name_str; +pub use codex_rollout::rollout_date_parts; impl codex_rollout::RolloutConfigView for Config { fn codex_home(&self) -> &std::path::Path { @@ -22,21 +36,6 @@ impl codex_rollout::RolloutConfigView for Config { } } -pub use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; -pub use codex_rollout::INTERACTIVE_SESSION_SOURCES; -pub use codex_rollout::RolloutRecorder; -pub use codex_rollout::RolloutRecorderParams; -pub use codex_rollout::SESSIONS_SUBDIR; -pub use codex_rollout::SessionMeta; -pub use codex_rollout::append_thread_name; -pub use codex_rollout::find_archived_thread_path_by_id_str; -#[deprecated(note = "use find_thread_path_by_id_str")] -pub use codex_rollout::find_conversation_path_by_id_str; -pub use codex_rollout::find_thread_name_by_id; -pub use codex_rollout::find_thread_path_by_id_str; -pub use codex_rollout::find_thread_path_by_name_str; -pub use codex_rollout::rollout_date_parts; - pub mod list { pub use codex_rollout::list::*; } diff --git a/codex-rs/core/src/state_db_bridge.rs b/codex-rs/core/src/state_db_bridge.rs index 9dc6c18a678..1465299f393 100644 --- a/codex-rs/core/src/state_db_bridge.rs +++ b/codex-rs/core/src/state_db_bridge.rs @@ -1,17 +1,17 @@ -use codex_rollout::db as rollout_state_db; -pub use codex_rollout::db::StateDbHandle; -pub use codex_rollout::db::apply_rollout_items; -pub use codex_rollout::db::find_rollout_path_by_id; -pub use codex_rollout::db::get_dynamic_tools; -pub use codex_rollout::db::list_thread_ids_db; -pub use codex_rollout::db::list_threads_db; -pub use codex_rollout::db::mark_thread_memory_mode_polluted; -pub use codex_rollout::db::normalize_cwd_for_state_db; -pub use codex_rollout::db::open_if_present; -pub use codex_rollout::db::persist_dynamic_tools; -pub use codex_rollout::db::read_repair_rollout_path; -pub use codex_rollout::db::reconcile_rollout; -pub use codex_rollout::db::touch_thread_updated_at; +use codex_rollout::state_db as rollout_state_db; +pub use codex_rollout::state_db::StateDbHandle; +pub use codex_rollout::state_db::apply_rollout_items; +pub use codex_rollout::state_db::find_rollout_path_by_id; +pub use codex_rollout::state_db::get_dynamic_tools; +pub use codex_rollout::state_db::list_thread_ids_db; +pub use codex_rollout::state_db::list_threads_db; +pub use codex_rollout::state_db::mark_thread_memory_mode_polluted; +pub use codex_rollout::state_db::normalize_cwd_for_state_db; +pub use codex_rollout::state_db::open_if_present; +pub use codex_rollout::state_db::persist_dynamic_tools; +pub use codex_rollout::state_db::read_repair_rollout_path; +pub use codex_rollout::state_db::reconcile_rollout; +pub use codex_rollout::state_db::touch_thread_updated_at; pub use codex_state::LogEntry; use crate::config::Config; diff --git a/codex-rs/rollout/Cargo.toml b/codex-rs/rollout/Cargo.toml index 8564c71c88e..ef5a8dc22a8 100644 --- a/codex-rs/rollout/Cargo.toml +++ b/codex-rs/rollout/Cargo.toml @@ -22,6 +22,7 @@ codex-login = { workspace = true } codex-otel = { workspace = true } codex-protocol = { workspace = true } codex-state = { workspace = true } +codex-utils-path = { workspace = true } codex-utils-string = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/codex-rs/rollout/src/lib.rs b/codex-rs/rollout/src/lib.rs index fca9b447a05..160792a3901 100644 --- a/codex-rs/rollout/src/lib.rs +++ b/codex-rs/rollout/src/lib.rs @@ -5,22 +5,18 @@ use std::sync::LazyLock; use codex_protocol::protocol::SessionSource; pub mod config; -#[path = "state_db.rs"] -pub mod db; pub mod list; pub mod metadata; pub mod policy; pub mod recorder; pub mod session_index; - -mod path_utils; +pub mod state_db; pub(crate) mod default_client { pub use codex_login::default_client::*; } pub(crate) use codex_protocol::protocol; -pub(crate) use db as state_db; pub const SESSIONS_SUBDIR: &str = "sessions"; pub const ARCHIVED_SESSIONS_SUBDIR: &str = "archived_sessions"; @@ -36,7 +32,6 @@ pub static INTERACTIVE_SESSION_SOURCES: LazyLock> = LazyLock: pub use codex_protocol::protocol::SessionMeta; pub use config::RolloutConfig; pub use config::RolloutConfigView; -pub use db::StateDbHandle; pub use list::find_archived_thread_path_by_id_str; pub use list::find_thread_path_by_id_str; #[deprecated(note = "use find_thread_path_by_id_str")] @@ -49,6 +44,7 @@ pub use session_index::append_thread_name; pub use session_index::find_thread_name_by_id; pub use session_index::find_thread_names_by_ids; pub use session_index::find_thread_path_by_name_str; +pub use state_db::StateDbHandle; #[cfg(test)] mod tests; diff --git a/codex-rs/rollout/src/path_utils.rs b/codex-rs/rollout/src/path_utils.rs deleted file mode 100644 index 561b69e5218..00000000000 --- a/codex-rs/rollout/src/path_utils.rs +++ /dev/null @@ -1,97 +0,0 @@ -use std::path::Path; -use std::path::PathBuf; - -pub fn normalize_for_path_comparison(path: impl AsRef) -> std::io::Result { - let canonical = path.as_ref().canonicalize()?; - Ok(normalize_for_wsl(canonical)) -} - -fn normalize_for_wsl(path: PathBuf) -> PathBuf { - normalize_for_wsl_with_flag(path, is_wsl()) -} - -fn is_wsl() -> bool { - #[cfg(target_os = "linux")] - { - if std::env::var_os("WSL_DISTRO_NAME").is_some() { - return true; - } - match std::fs::read_to_string("/proc/version") { - Ok(version) => version.to_lowercase().contains("microsoft"), - Err(_) => false, - } - } - #[cfg(not(target_os = "linux"))] - { - false - } -} - -fn normalize_for_wsl_with_flag(path: PathBuf, is_wsl: bool) -> PathBuf { - if !is_wsl { - return path; - } - - if !is_wsl_case_insensitive_path(&path) { - return path; - } - - lower_ascii_path(path) -} - -fn is_wsl_case_insensitive_path(path: &Path) -> bool { - #[cfg(target_os = "linux")] - { - use std::os::unix::ffi::OsStrExt; - use std::path::Component; - - let mut components = path.components(); - let Some(Component::RootDir) = components.next() else { - return false; - }; - let Some(Component::Normal(mnt)) = components.next() else { - return false; - }; - if !ascii_eq_ignore_case(mnt.as_bytes(), b"mnt") { - return false; - } - let Some(Component::Normal(drive)) = components.next() else { - return false; - }; - let drive_bytes = drive.as_bytes(); - drive_bytes.len() == 1 && drive_bytes[0].is_ascii_alphabetic() - } - #[cfg(not(target_os = "linux"))] - { - let _ = path; - false - } -} - -#[cfg(target_os = "linux")] -fn ascii_eq_ignore_case(left: &[u8], right: &[u8]) -> bool { - left.len() == right.len() - && left - .iter() - .zip(right) - .all(|(lhs, rhs)| lhs.to_ascii_lowercase() == *rhs) -} - -#[cfg(target_os = "linux")] -fn lower_ascii_path(path: PathBuf) -> PathBuf { - use std::ffi::OsString; - use std::os::unix::ffi::OsStrExt; - use std::os::unix::ffi::OsStringExt; - - let bytes = path.as_os_str().as_bytes(); - let mut lowered = Vec::with_capacity(bytes.len()); - for byte in bytes { - lowered.push(byte.to_ascii_lowercase()); - } - PathBuf::from(OsString::from_vec(lowered)) -} - -#[cfg(not(target_os = "linux"))] -fn lower_ascii_path(path: PathBuf) -> PathBuf { - path -} diff --git a/codex-rs/rollout/src/recorder.rs b/codex-rs/rollout/src/recorder.rs index 6a5ef852dd7..f39c38af603 100644 --- a/codex-rs/rollout/src/recorder.rs +++ b/codex-rs/rollout/src/recorder.rs @@ -42,7 +42,6 @@ use super::policy::EventPersistenceMode; use super::policy::is_persisted_response_item; use crate::config::RolloutConfigView; use crate::default_client::originator; -use crate::path_utils; use crate::state_db; use crate::state_db::StateDbHandle; use codex_git_utils::collect_git_info; @@ -57,8 +56,7 @@ use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_state::StateRuntime; use codex_state::ThreadMetadataBuilder; -use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_output_truncation::truncate_text; +use codex_utils_path as path_utils; /// Records all [`ResponseItem`]s for a session and flushes them to disk after /// every update. diff --git a/codex-rs/rollout/src/state_db.rs b/codex-rs/rollout/src/state_db.rs index 2e0485f1814..6367a27ca93 100644 --- a/codex-rs/rollout/src/state_db.rs +++ b/codex-rs/rollout/src/state_db.rs @@ -3,7 +3,6 @@ use crate::config::RolloutConfigView; use crate::list::Cursor; use crate::list::ThreadSortKey; use crate::metadata; -use crate::path_utils::normalize_for_path_comparison; use chrono::DateTime; use chrono::NaiveDateTime; use chrono::Timelike; @@ -14,6 +13,7 @@ use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionSource; pub use codex_state::LogEntry; use codex_state::ThreadMetadataBuilder; +use codex_utils_path::normalize_for_path_comparison; use serde_json::Value; use std::path::Path; use std::path::PathBuf; From d70dfd7e027d9a9a0f19d1feec1a9ba1c26081cf Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 16:14:53 -0700 Subject: [PATCH 08/14] fix --- codex-rs/core/src/utils/mod.rs | 1 + codex-rs/core/src/utils/path_utils.rs | 1 + codex-rs/utils/path-utils/BUILD.bazel | 6 + codex-rs/utils/path-utils/Cargo.toml | 17 ++ codex-rs/utils/path-utils/src/lib.rs | 224 ++++++++++++++++++ .../utils/path-utils/src/path_utils_tests.rs | 78 ++++++ 6 files changed, 327 insertions(+) create mode 100644 codex-rs/core/src/utils/mod.rs create mode 100644 codex-rs/core/src/utils/path_utils.rs create mode 100644 codex-rs/utils/path-utils/BUILD.bazel create mode 100644 codex-rs/utils/path-utils/Cargo.toml create mode 100644 codex-rs/utils/path-utils/src/lib.rs create mode 100644 codex-rs/utils/path-utils/src/path_utils_tests.rs diff --git a/codex-rs/core/src/utils/mod.rs b/codex-rs/core/src/utils/mod.rs new file mode 100644 index 00000000000..adbdcef12ba --- /dev/null +++ b/codex-rs/core/src/utils/mod.rs @@ -0,0 +1 @@ +pub mod path_utils; diff --git a/codex-rs/core/src/utils/path_utils.rs b/codex-rs/core/src/utils/path_utils.rs new file mode 100644 index 00000000000..3715857d1a5 --- /dev/null +++ b/codex-rs/core/src/utils/path_utils.rs @@ -0,0 +1 @@ +pub use codex_utils_path::*; diff --git a/codex-rs/utils/path-utils/BUILD.bazel b/codex-rs/utils/path-utils/BUILD.bazel new file mode 100644 index 00000000000..fdfbf099b1a --- /dev/null +++ b/codex-rs/utils/path-utils/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "path-utils", + crate_name = "codex_utils_path", +) diff --git a/codex-rs/utils/path-utils/Cargo.toml b/codex-rs/utils/path-utils/Cargo.toml new file mode 100644 index 00000000000..0d1693361f8 --- /dev/null +++ b/codex-rs/utils/path-utils/Cargo.toml @@ -0,0 +1,17 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-utils-path" +version.workspace = true + +[lints] +workspace = true + +[dependencies] +codex-utils-absolute-path = { workspace = true } +dunce = { workspace = true } +tempfile = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tempfile = { workspace = true } diff --git a/codex-rs/utils/path-utils/src/lib.rs b/codex-rs/utils/path-utils/src/lib.rs new file mode 100644 index 00000000000..4ec1987f4be --- /dev/null +++ b/codex-rs/utils/path-utils/src/lib.rs @@ -0,0 +1,224 @@ +//! Path normalization, symlink resolution, and atomic writes shared across Codex crates. + +use codex_utils_absolute_path::AbsolutePathBuf; +use std::collections::HashSet; +use std::io; +use std::path::Path; +use std::path::PathBuf; +use tempfile::NamedTempFile; + +/// Returns true when the current process is running under Windows Subsystem for Linux. +/// +/// Matches the WSL detection logic in `codex-core`'s `env` module so this crate stays +/// independent of `codex-core`. +fn is_wsl() -> bool { + #[cfg(target_os = "linux")] + { + if std::env::var_os("WSL_DISTRO_NAME").is_some() { + return true; + } + match std::fs::read_to_string("/proc/version") { + Ok(version) => version.to_lowercase().contains("microsoft"), + Err(_) => false, + } + } + #[cfg(not(target_os = "linux"))] + { + false + } +} + +pub fn normalize_for_path_comparison(path: impl AsRef) -> std::io::Result { + let canonical = path.as_ref().canonicalize()?; + Ok(normalize_for_wsl(canonical)) +} + +pub fn normalize_for_native_workdir(path: impl AsRef) -> PathBuf { + normalize_for_native_workdir_with_flag(path.as_ref().to_path_buf(), cfg!(windows)) +} + +pub struct SymlinkWritePaths { + pub read_path: Option, + pub write_path: PathBuf, +} + +/// Resolve the final filesystem target for `path` while retaining a safe write path. +/// +/// This follows symlink chains (including relative symlink targets) until it reaches a +/// non-symlink path. If the chain cycles or any metadata/link resolution fails, it +/// returns `read_path: None` and uses the original absolute path as `write_path`. +/// There is no fixed max-resolution count; cycles are detected via a visited set. +pub fn resolve_symlink_write_paths(path: &Path) -> io::Result { + let root = AbsolutePathBuf::from_absolute_path(path) + .map(AbsolutePathBuf::into_path_buf) + .unwrap_or_else(|_| path.to_path_buf()); + let mut current = root.clone(); + let mut visited = HashSet::new(); + + // Follow symlink chains while guarding against cycles. + loop { + let meta = match std::fs::symlink_metadata(¤t) { + Ok(meta) => meta, + Err(err) if err.kind() == io::ErrorKind::NotFound => { + return Ok(SymlinkWritePaths { + read_path: Some(current.clone()), + write_path: current, + }); + } + Err(_) => { + return Ok(SymlinkWritePaths { + read_path: None, + write_path: root, + }); + } + }; + + if !meta.file_type().is_symlink() { + return Ok(SymlinkWritePaths { + read_path: Some(current.clone()), + write_path: current, + }); + } + + // If we've already seen this path, the chain cycles. + if !visited.insert(current.clone()) { + return Ok(SymlinkWritePaths { + read_path: None, + write_path: root, + }); + } + + let target = match std::fs::read_link(¤t) { + Ok(target) => target, + Err(_) => { + return Ok(SymlinkWritePaths { + read_path: None, + write_path: root, + }); + } + }; + + let next = if target.is_absolute() { + AbsolutePathBuf::from_absolute_path(&target) + } else if let Some(parent) = current.parent() { + AbsolutePathBuf::resolve_path_against_base(&target, parent) + } else { + return Ok(SymlinkWritePaths { + read_path: None, + write_path: root, + }); + }; + + let next = match next { + Ok(path) => path.into_path_buf(), + Err(_) => { + return Ok(SymlinkWritePaths { + read_path: None, + write_path: root, + }); + } + }; + + current = next; + } +} + +pub fn write_atomically(write_path: &Path, contents: &str) -> io::Result<()> { + let parent = write_path.parent().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("path {} has no parent directory", write_path.display()), + ) + })?; + std::fs::create_dir_all(parent)?; + let tmp = NamedTempFile::new_in(parent)?; + std::fs::write(tmp.path(), contents)?; + tmp.persist(write_path)?; + Ok(()) +} + +fn normalize_for_wsl(path: PathBuf) -> PathBuf { + normalize_for_wsl_with_flag(path, is_wsl()) +} + +fn normalize_for_native_workdir_with_flag(path: PathBuf, is_windows: bool) -> PathBuf { + if is_windows { + dunce::simplified(&path).to_path_buf() + } else { + path + } +} + +fn normalize_for_wsl_with_flag(path: PathBuf, is_wsl: bool) -> PathBuf { + if !is_wsl { + return path; + } + + if !is_wsl_case_insensitive_path(&path) { + return path; + } + + lower_ascii_path(path) +} + +fn is_wsl_case_insensitive_path(path: &Path) -> bool { + #[cfg(target_os = "linux")] + { + use std::os::unix::ffi::OsStrExt; + use std::path::Component; + + let mut components = path.components(); + let Some(Component::RootDir) = components.next() else { + return false; + }; + let Some(Component::Normal(mnt)) = components.next() else { + return false; + }; + if !ascii_eq_ignore_case(mnt.as_bytes(), b"mnt") { + return false; + } + let Some(Component::Normal(drive)) = components.next() else { + return false; + }; + let drive_bytes = drive.as_bytes(); + drive_bytes.len() == 1 && drive_bytes[0].is_ascii_alphabetic() + } + #[cfg(not(target_os = "linux"))] + { + let _ = path; + false + } +} + +#[cfg(target_os = "linux")] +fn ascii_eq_ignore_case(left: &[u8], right: &[u8]) -> bool { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(lhs, rhs)| lhs.to_ascii_lowercase() == *rhs) +} + +#[cfg(target_os = "linux")] +fn lower_ascii_path(path: PathBuf) -> PathBuf { + use std::ffi::OsString; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::ffi::OsStringExt; + + // WSL mounts Windows drives under /mnt/, which are case-insensitive. + let bytes = path.as_os_str().as_bytes(); + let mut lowered = Vec::with_capacity(bytes.len()); + for byte in bytes { + lowered.push(byte.to_ascii_lowercase()); + } + PathBuf::from(OsString::from_vec(lowered)) +} + +#[cfg(not(target_os = "linux"))] +fn lower_ascii_path(path: PathBuf) -> PathBuf { + path +} + +#[cfg(test)] +#[path = "path_utils_tests.rs"] +mod tests; diff --git a/codex-rs/utils/path-utils/src/path_utils_tests.rs b/codex-rs/utils/path-utils/src/path_utils_tests.rs new file mode 100644 index 00000000000..f028133f12a --- /dev/null +++ b/codex-rs/utils/path-utils/src/path_utils_tests.rs @@ -0,0 +1,78 @@ +#[cfg(unix)] +mod symlinks { + use super::super::resolve_symlink_write_paths; + use pretty_assertions::assert_eq; + use std::os::unix::fs::symlink; + + #[test] + fn symlink_cycles_fall_back_to_root_write_path() -> std::io::Result<()> { + let dir = tempfile::tempdir()?; + let a = dir.path().join("a"); + let b = dir.path().join("b"); + + symlink(&b, &a)?; + symlink(&a, &b)?; + + let resolved = resolve_symlink_write_paths(&a)?; + + assert_eq!(resolved.read_path, None); + assert_eq!(resolved.write_path, a); + Ok(()) + } +} + +#[cfg(target_os = "linux")] +mod wsl { + use super::super::normalize_for_wsl_with_flag; + use pretty_assertions::assert_eq; + use std::path::PathBuf; + + #[test] + fn wsl_mnt_drive_paths_lowercase() { + let normalized = normalize_for_wsl_with_flag(PathBuf::from("/mnt/C/Users/Dev"), true); + + assert_eq!(normalized, PathBuf::from("/mnt/c/users/dev")); + } + + #[test] + fn wsl_non_drive_paths_unchanged() { + let path = PathBuf::from("/mnt/cc/Users/Dev"); + let normalized = normalize_for_wsl_with_flag(path.clone(), true); + + assert_eq!(normalized, path); + } + + #[test] + fn wsl_non_mnt_paths_unchanged() { + let path = PathBuf::from("/home/Dev"); + let normalized = normalize_for_wsl_with_flag(path.clone(), true); + + assert_eq!(normalized, path); + } +} + +mod native_workdir { + use super::super::normalize_for_native_workdir_with_flag; + use pretty_assertions::assert_eq; + use std::path::PathBuf; + + #[cfg(target_os = "windows")] + #[test] + fn windows_verbatim_paths_are_simplified() { + let path = PathBuf::from(r"\\?\D:\c\x\worktrees\2508\swift-base"); + let normalized = normalize_for_native_workdir_with_flag(path, true); + + assert_eq!( + normalized, + PathBuf::from(r"D:\c\x\worktrees\2508\swift-base") + ); + } + + #[test] + fn non_windows_paths_are_unchanged() { + let path = PathBuf::from(r"\\?\D:\c\x\worktrees\2508\swift-base"); + let normalized = normalize_for_native_workdir_with_flag(path.clone(), false); + + assert_eq!(normalized, path); + } +} From 7d1a4bdce5edfb067e87cbafd8a00d92126c1621 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 16:27:24 -0700 Subject: [PATCH 09/14] fix --- codex-rs/core/src/env.rs | 46 ---------------------------- codex-rs/core/src/lib.rs | 2 +- codex-rs/utils/path-utils/src/lib.rs | 25 ++------------- 3 files changed, 4 insertions(+), 69 deletions(-) delete mode 100644 codex-rs/core/src/env.rs diff --git a/codex-rs/core/src/env.rs b/codex-rs/core/src/env.rs deleted file mode 100644 index c99b2427774..00000000000 --- a/codex-rs/core/src/env.rs +++ /dev/null @@ -1,46 +0,0 @@ -//! Functions for environment detection that need to be shared across crates. - -fn env_var_set(key: &str) -> bool { - std::env::var(key).is_ok_and(|v| !v.trim().is_empty()) -} - -/// Returns true if the current process is running under Windows Subsystem for Linux. -pub fn is_wsl() -> bool { - #[cfg(target_os = "linux")] - { - if std::env::var_os("WSL_DISTRO_NAME").is_some() { - return true; - } - match std::fs::read_to_string("/proc/version") { - Ok(version) => version.to_lowercase().contains("microsoft"), - Err(_) => false, - } - } - #[cfg(not(target_os = "linux"))] - { - false - } -} - -/// Returns true when Codex is likely running in an environment without a usable GUI. -/// -/// This is intentionally conservative and is used by frontends to avoid flows that would try to -/// open a browser (e.g. device-code auth fallback). -pub fn is_headless_environment() -> bool { - if env_var_set("CI") - || env_var_set("SSH_CONNECTION") - || env_var_set("SSH_CLIENT") - || env_var_set("SSH_TTY") - { - return true; - } - - #[cfg(target_os = "linux")] - { - if !env_var_set("DISPLAY") && !env_var_set("WAYLAND_DISPLAY") { - return true; - } - } - - false -} diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 773b65fca7c..09232f3f82f 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -33,7 +33,7 @@ pub mod connectors; mod context_manager; mod contextual_user_message; pub mod custom_prompts; -pub mod env; +pub use codex_utils_path::env; mod environment_context; pub mod error; pub mod exec; diff --git a/codex-rs/utils/path-utils/src/lib.rs b/codex-rs/utils/path-utils/src/lib.rs index 4ec1987f4be..a6dc6f8b3a9 100644 --- a/codex-rs/utils/path-utils/src/lib.rs +++ b/codex-rs/utils/path-utils/src/lib.rs @@ -1,5 +1,7 @@ //! Path normalization, symlink resolution, and atomic writes shared across Codex crates. +pub mod env; + use codex_utils_absolute_path::AbsolutePathBuf; use std::collections::HashSet; use std::io; @@ -7,27 +9,6 @@ use std::path::Path; use std::path::PathBuf; use tempfile::NamedTempFile; -/// Returns true when the current process is running under Windows Subsystem for Linux. -/// -/// Matches the WSL detection logic in `codex-core`'s `env` module so this crate stays -/// independent of `codex-core`. -fn is_wsl() -> bool { - #[cfg(target_os = "linux")] - { - if std::env::var_os("WSL_DISTRO_NAME").is_some() { - return true; - } - match std::fs::read_to_string("/proc/version") { - Ok(version) => version.to_lowercase().contains("microsoft"), - Err(_) => false, - } - } - #[cfg(not(target_os = "linux"))] - { - false - } -} - pub fn normalize_for_path_comparison(path: impl AsRef) -> std::io::Result { let canonical = path.as_ref().canonicalize()?; Ok(normalize_for_wsl(canonical)) @@ -138,7 +119,7 @@ pub fn write_atomically(write_path: &Path, contents: &str) -> io::Result<()> { } fn normalize_for_wsl(path: PathBuf) -> PathBuf { - normalize_for_wsl_with_flag(path, is_wsl()) + normalize_for_wsl_with_flag(path, env::is_wsl()) } fn normalize_for_native_workdir_with_flag(path: PathBuf, is_windows: bool) -> PathBuf { From 1bbd4bd14e9cb896479a8517296f69130a909129 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 16:27:34 -0700 Subject: [PATCH 10/14] fix --- codex-rs/utils/path-utils/src/env.rs | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 codex-rs/utils/path-utils/src/env.rs diff --git a/codex-rs/utils/path-utils/src/env.rs b/codex-rs/utils/path-utils/src/env.rs new file mode 100644 index 00000000000..c99b2427774 --- /dev/null +++ b/codex-rs/utils/path-utils/src/env.rs @@ -0,0 +1,46 @@ +//! Functions for environment detection that need to be shared across crates. + +fn env_var_set(key: &str) -> bool { + std::env::var(key).is_ok_and(|v| !v.trim().is_empty()) +} + +/// Returns true if the current process is running under Windows Subsystem for Linux. +pub fn is_wsl() -> bool { + #[cfg(target_os = "linux")] + { + if std::env::var_os("WSL_DISTRO_NAME").is_some() { + return true; + } + match std::fs::read_to_string("/proc/version") { + Ok(version) => version.to_lowercase().contains("microsoft"), + Err(_) => false, + } + } + #[cfg(not(target_os = "linux"))] + { + false + } +} + +/// Returns true when Codex is likely running in an environment without a usable GUI. +/// +/// This is intentionally conservative and is used by frontends to avoid flows that would try to +/// open a browser (e.g. device-code auth fallback). +pub fn is_headless_environment() -> bool { + if env_var_set("CI") + || env_var_set("SSH_CONNECTION") + || env_var_set("SSH_CLIENT") + || env_var_set("SSH_TTY") + { + return true; + } + + #[cfg(target_os = "linux")] + { + if !env_var_set("DISPLAY") && !env_var_set("WAYLAND_DISPLAY") { + return true; + } + } + + false +} From e10b3108b7901f7d5f2f897dba296277d7b464d8 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 16:43:49 -0700 Subject: [PATCH 11/14] Remove unrelated PR cleanup changes Co-authored-by: Codex --- codex-rs/app-server/src/codex_message_processor.rs | 7 +++++++ codex-rs/app-server/src/in_process.rs | 1 + codex-rs/app-server/src/message_processor.rs | 4 ++++ codex-rs/core/src/context_manager/history.rs | 2 -- codex-rs/core/src/context_manager/history_tests.rs | 2 -- codex-rs/core/src/lib.rs | 1 - codex-rs/core/src/thread_rollout_truncation_tests.rs | 2 -- codex-rs/protocol/src/lib.rs | 2 -- 8 files changed, 12 insertions(+), 9 deletions(-) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 02d7ebe13c7..1d58e7317d7 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -1933,6 +1933,13 @@ impl CodexMessageProcessor { } } + pub(crate) async fn cancel_active_login(&self) { + let mut guard = self.active_login.lock().await; + if let Some(active_login) = guard.take() { + drop(active_login); + } + } + pub(crate) async fn clear_all_thread_listeners(&self) { self.thread_state_manager.clear_all_listeners().await; } diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index 46495c50145..79f32c90831 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -458,6 +458,7 @@ fn start_uninitialized(args: InProcessStartArgs) -> InProcessClientHandle { } processor.clear_runtime_references(); + processor.cancel_active_login().await; processor.connection_closed(IN_PROCESS_CONNECTION_ID).await; processor.clear_all_thread_listeners().await; processor.drain_background_tasks().await; diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 5831a357e6e..a67c810bd0e 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -458,6 +458,10 @@ impl MessageProcessor { self.codex_message_processor.drain_background_tasks().await; } + pub(crate) async fn cancel_active_login(&self) { + self.codex_message_processor.cancel_active_login().await; + } + pub(crate) async fn clear_all_thread_listeners(&self) { self.codex_message_processor .clear_all_thread_listeners() diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 1d946f183c3..ec2df30d3bd 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1,5 +1,3 @@ -#![allow(warnings, clippy::all)] - use crate::codex::TurnContext; use crate::context_manager::normalize; use crate::event_mapping::has_non_contextual_dev_message_content; diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 80a542bbe03..3c508e05ff7 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -1,5 +1,3 @@ -#![allow(warnings, clippy::all)] - use super::*; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 09232f3f82f..71f4328aede 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -3,7 +3,6 @@ // Prevent accidental direct writes to stdout/stderr in library code. All // user-visible output must go through the appropriate abstraction (e.g., // the TUI or the tracing stack). -#![allow(warnings, clippy::all)] #![deny(clippy::print_stdout, clippy::print_stderr)] mod analytics_client; diff --git a/codex-rs/core/src/thread_rollout_truncation_tests.rs b/codex-rs/core/src/thread_rollout_truncation_tests.rs index 5549a198e53..20f65cd9aa3 100644 --- a/codex-rs/core/src/thread_rollout_truncation_tests.rs +++ b/codex-rs/core/src/thread_rollout_truncation_tests.rs @@ -1,5 +1,3 @@ -#![allow(warnings, clippy::all)] - use super::*; use crate::codex::make_session_and_context; use codex_protocol::models::ContentItem; diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index fcbfcd89033..56924cc50d9 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -1,5 +1,3 @@ -#![allow(warnings, clippy::all)] - pub mod account; mod agent_path; mod thread_id; From 47b533abc171b2c639ab3c8a44c7f3c92bac71d4 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 16:46:47 -0700 Subject: [PATCH 12/14] Revert unrelated tui auth changes Co-authored-by: Codex --- codex-rs/tui/src/lib.rs | 51 ++++++-- codex-rs/tui_app_server/src/lib.rs | 49 ++++++-- .../tui_app_server/src/onboarding/auth.rs | 117 +++++++++++++----- .../onboarding/auth/headless_chatgpt_login.rs | 2 + .../src/onboarding/onboarding_screen.rs | 9 ++ 5 files changed, 180 insertions(+), 48 deletions(-) diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 54162006911..039148a7186 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -609,6 +609,7 @@ async fn run_ratatui_app( terminal.clear()?; let mut tui = Tui::new(terminal); + let mut terminal_restore_guard = TerminalRestoreGuard::new(); #[cfg(not(debug_assertions))] { @@ -619,7 +620,7 @@ async fn run_ratatui_app( match update_prompt::run_update_prompt_if_needed(&mut tui, &initial_config).await? { UpdatePromptOutcome::Continue => {} UpdatePromptOutcome::RunUpdate(action) => { - crate::tui::restore()?; + terminal_restore_guard.restore()?; return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), thread_id: None, @@ -660,7 +661,7 @@ async fn run_ratatui_app( ) .await?; if onboarding_result.should_exit { - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); let _ = tui.terminal.clear(); return Ok(AppExitInfo { @@ -701,7 +702,7 @@ async fn run_ratatui_app( let mut missing_session_exit = |id_str: &str, action: &str| { error!("Error finding conversation path: {id_str}"); - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); let _ = tui.terminal.clear(); Ok(AppExitInfo { @@ -773,7 +774,7 @@ async fn run_ratatui_app( error!( "Error reading session metadata from latest rollout: {rollout_path}" ); - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); let _ = tui.terminal.clear(); return Ok(AppExitInfo { @@ -795,7 +796,7 @@ async fn run_ratatui_app( } else if cli.fork_picker { match resume_picker::run_fork_picker(&mut tui, &config, cli.fork_show_all).await? { resume_picker::SessionSelection::Exit => { - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -867,7 +868,7 @@ async fn run_ratatui_app( error!( "Error reading session metadata from latest rollout: {rollout_path}" ); - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); let _ = tui.terminal.clear(); return Ok(AppExitInfo { @@ -887,7 +888,7 @@ async fn run_ratatui_app( } else if cli.resume_picker { match resume_picker::run_resume_picker(&mut tui, &config, cli.resume_show_all).await? { resume_picker::SessionSelection::Exit => { - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -929,7 +930,7 @@ async fn run_ratatui_app( { ResolveCwdOutcome::Continue(cwd) => cwd, ResolveCwdOutcome::Exit => { - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -1003,7 +1004,7 @@ async fn run_ratatui_app( ) .await; - restore(); + terminal_restore_guard.restore_silently(); // Mark the end of the recorded session. session_log::log_session_end(); // ignore error when collecting usage – report underlying error instead @@ -1128,6 +1129,38 @@ fn restore() { } } +struct TerminalRestoreGuard { + active: bool, +} + +impl TerminalRestoreGuard { + fn new() -> Self { + Self { active: true } + } + + #[cfg_attr(debug_assertions, allow(dead_code))] + fn restore(&mut self) -> color_eyre::Result<()> { + if self.active { + crate::tui::restore()?; + self.active = false; + } + Ok(()) + } + + fn restore_silently(&mut self) { + if self.active { + restore(); + self.active = false; + } + } +} + +impl Drop for TerminalRestoreGuard { + fn drop(&mut self) { + self.restore_silently(); + } +} + /// Determine whether to use the terminal's alternate screen buffer. /// /// The alternate screen buffer provides a cleaner fullscreen experience without polluting diff --git a/codex-rs/tui_app_server/src/lib.rs b/codex-rs/tui_app_server/src/lib.rs index 17e309d5fbd..790a7c6f55c 100644 --- a/codex-rs/tui_app_server/src/lib.rs +++ b/codex-rs/tui_app_server/src/lib.rs @@ -938,6 +938,7 @@ async fn run_ratatui_app( terminal.clear()?; let mut tui = Tui::new(terminal); + let mut terminal_restore_guard = TerminalRestoreGuard::new(); #[cfg(not(debug_assertions))] { @@ -948,7 +949,7 @@ async fn run_ratatui_app( match update_prompt::run_update_prompt_if_needed(&mut tui, &initial_config).await? { UpdatePromptOutcome::Continue => {} UpdatePromptOutcome::RunUpdate(action) => { - crate::tui::restore()?; + terminal_restore_guard.restore()?; return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), thread_id: None, @@ -1016,7 +1017,7 @@ async fn run_ratatui_app( ) .await?; if onboarding_result.should_exit { - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); let _ = tui.terminal.clear(); return Ok(AppExitInfo { @@ -1062,7 +1063,7 @@ async fn run_ratatui_app( let mut missing_session_exit = |id_str: &str, action: &str| { error!("Error finding conversation path: {id_str}"); - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); let _ = tui.terminal.clear(); Ok(AppExitInfo { @@ -1137,7 +1138,7 @@ async fn run_ratatui_app( .await? { resume_picker::SessionSelection::Exit => { - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -1189,7 +1190,7 @@ async fn run_ratatui_app( .await? { resume_picker::SessionSelection::Exit => { - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -1235,7 +1236,7 @@ async fn run_ratatui_app( { ResolveCwdOutcome::Continue(cwd) => cwd, ResolveCwdOutcome::Exit => { - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); return Ok(AppExitInfo { token_usage: codex_protocol::protocol::TokenUsage::default(), @@ -1303,7 +1304,7 @@ async fn run_ratatui_app( { Ok(app_server) => app_server, Err(err) => { - restore(); + terminal_restore_guard.restore_silently(); session_log::log_session_end(); return Err(err); } @@ -1326,7 +1327,7 @@ async fn run_ratatui_app( ) .await; - restore(); + terminal_restore_guard.restore_silently(); // Mark the end of the recorded session. session_log::log_session_end(); // ignore error when collecting usage – report underlying error instead @@ -1468,6 +1469,38 @@ fn restore() { } } +struct TerminalRestoreGuard { + active: bool, +} + +impl TerminalRestoreGuard { + fn new() -> Self { + Self { active: true } + } + + #[cfg_attr(debug_assertions, allow(dead_code))] + fn restore(&mut self) -> color_eyre::Result<()> { + if self.active { + crate::tui::restore()?; + self.active = false; + } + Ok(()) + } + + fn restore_silently(&mut self) { + if self.active { + restore(); + self.active = false; + } + } +} + +impl Drop for TerminalRestoreGuard { + fn drop(&mut self) { + self.restore_silently(); + } +} + /// Determine whether to use the terminal's alternate screen buffer. /// /// The alternate screen buffer provides a cleaner fullscreen experience without polluting diff --git a/codex-rs/tui_app_server/src/onboarding/auth.rs b/codex-rs/tui_app_server/src/onboarding/auth.rs index 612fdbe2ace..37f8819b62f 100644 --- a/codex-rs/tui_app_server/src/onboarding/auth.rs +++ b/codex-rs/tui_app_server/src/onboarding/auth.rs @@ -163,37 +163,7 @@ impl KeyboardHandler for AuthModeWidget { } KeyCode::Esc => { tracing::info!("Esc pressed"); - let mut sign_in_state = self.sign_in_state.write().unwrap(); - match &*sign_in_state { - SignInState::ChatGptContinueInBrowser(state) => { - let request_handle = self.app_server_request_handle.clone(); - let login_id = state.login_id.clone(); - tokio::spawn(async move { - let _ = request_handle - .request_typed::( - ClientRequest::CancelLoginAccount { - request_id: onboarding_request_id(), - params: CancelLoginAccountParams { login_id }, - }, - ) - .await; - }); - *sign_in_state = SignInState::PickMode; - drop(sign_in_state); - self.set_error(/*message*/ None); - self.request_frame.schedule_frame(); - } - SignInState::ChatGptDeviceCode(state) => { - if let Some(cancel) = &state.cancel { - cancel.notify_one(); - } - *sign_in_state = SignInState::PickMode; - drop(sign_in_state); - self.set_error(/*message*/ None); - self.request_frame.schedule_frame(); - } - _ => {} - } + self.cancel_active_attempt(); } _ => {} } @@ -221,6 +191,36 @@ pub(crate) struct AuthModeWidget { } impl AuthModeWidget { + pub(crate) fn cancel_active_attempt(&self) { + let mut sign_in_state = self.sign_in_state.write().unwrap(); + match &*sign_in_state { + SignInState::ChatGptContinueInBrowser(state) => { + let request_handle = self.app_server_request_handle.clone(); + let login_id = state.login_id.clone(); + tokio::spawn(async move { + let _ = request_handle + .request_typed::( + ClientRequest::CancelLoginAccount { + request_id: onboarding_request_id(), + params: CancelLoginAccountParams { login_id }, + }, + ) + .await; + }); + } + SignInState::ChatGptDeviceCode(state) => { + if let Some(cancel) = &state.cancel { + cancel.notify_one(); + } + } + _ => return, + } + *sign_in_state = SignInState::PickMode; + drop(sign_in_state); + self.set_error(/*message*/ None); + self.request_frame.schedule_frame(); + } + fn set_error(&self, message: Option) { *self.error.write().unwrap() = message; } @@ -771,6 +771,7 @@ impl AuthModeWidget { .await { Ok(LoginAccountResponse::Chatgpt { login_id, auth_url }) => { + maybe_open_auth_url_in_browser(&request_handle, &auth_url); *error.write().unwrap() = None; *sign_in_state.write().unwrap() = SignInState::ChatGptContinueInBrowser(ContinueInBrowserState { @@ -880,6 +881,16 @@ impl WidgetRef for AuthModeWidget { } } +pub(super) fn maybe_open_auth_url_in_browser(request_handle: &AppServerRequestHandle, url: &str) { + if !matches!(request_handle, AppServerRequestHandle::InProcess(_)) { + return; + } + + if let Err(err) = webbrowser::open(url) { + tracing::warn!("failed to open browser for login URL: {err}"); + } +} + #[cfg(test)] mod tests { use super::*; @@ -990,6 +1001,50 @@ mod tests { )); } + #[tokio::test] + async fn cancel_active_attempt_resets_browser_login_state() { + let (widget, _tmp) = widget_forced_chatgpt().await; + *widget.error.write().unwrap() = Some("still logging in".to_string()); + *widget.sign_in_state.write().unwrap() = + SignInState::ChatGptContinueInBrowser(ContinueInBrowserState { + login_id: "login-1".to_string(), + auth_url: "https://auth.example.com".to_string(), + }); + + widget.cancel_active_attempt(); + + assert_eq!(widget.error_message(), None); + assert!(matches!( + &*widget.sign_in_state.read().unwrap(), + SignInState::PickMode + )); + } + + #[tokio::test] + async fn cancel_active_attempt_notifies_device_code_login() { + let (widget, _tmp) = widget_forced_chatgpt().await; + let cancel = Arc::new(Notify::new()); + *widget.error.write().unwrap() = Some("still logging in".to_string()); + *widget.sign_in_state.write().unwrap() = + SignInState::ChatGptDeviceCode(ContinueWithDeviceCodeState { + device_code: None, + cancel: Some(cancel.clone()), + }); + + widget.cancel_active_attempt(); + + assert_eq!(widget.error_message(), None); + assert!(matches!( + &*widget.sign_in_state.read().unwrap(), + SignInState::PickMode + )); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), cancel.notified()) + .await + .is_ok() + ); + } + /// Collects all buffer cell symbols that contain the OSC 8 open sequence /// for the given URL. Returns the concatenated "inner" characters. fn collect_osc8_chars(buf: &Buffer, area: Rect, url: &str) -> String { diff --git a/codex-rs/tui_app_server/src/onboarding/auth/headless_chatgpt_login.rs b/codex-rs/tui_app_server/src/onboarding/auth/headless_chatgpt_login.rs index 33afe740b82..d0834fe1662 100644 --- a/codex-rs/tui_app_server/src/onboarding/auth/headless_chatgpt_login.rs +++ b/codex-rs/tui_app_server/src/onboarding/auth/headless_chatgpt_login.rs @@ -29,6 +29,7 @@ use super::ContinueInBrowserState; use super::ContinueWithDeviceCodeState; use super::SignInState; use super::mark_url_hyperlink; +use super::maybe_open_auth_url_in_browser; use super::onboarding_request_id; pub(super) fn start_headless_chatgpt_login(widget: &mut AuthModeWidget) { @@ -289,6 +290,7 @@ async fn fallback_to_browser_login( .await { Ok(LoginAccountResponse::Chatgpt { login_id, auth_url }) => { + maybe_open_auth_url_in_browser(&request_handle, &auth_url); *error.write().unwrap() = None; let _updated = set_device_code_state_for_active_attempt( &sign_in_state, diff --git a/codex-rs/tui_app_server/src/onboarding/onboarding_screen.rs b/codex-rs/tui_app_server/src/onboarding/onboarding_screen.rs index b6afd2cb984..d092d06e2f9 100644 --- a/codex-rs/tui_app_server/src/onboarding/onboarding_screen.rs +++ b/codex-rs/tui_app_server/src/onboarding/onboarding_screen.rs @@ -206,6 +206,14 @@ impl OnboardingScreen { self.should_exit } + fn cancel_auth_if_active(&self) { + for step in &self.steps { + if let Step::Auth(widget) = step { + widget.cancel_active_attempt(); + } + } + } + fn auth_widget_mut(&mut self) -> Option<&mut AuthModeWidget> { self.steps.iter_mut().find_map(|step| match step { Step::Auth(widget) => Some(widget), @@ -270,6 +278,7 @@ impl KeyboardHandler for OnboardingScreen { }; if should_quit { if self.is_auth_in_progress() { + self.cancel_auth_if_active(); // If the user cancels the auth menu, exit the app rather than // leave the user at a prompt in an unauthed state. self.should_exit = true; From bd4e05762945f4c4758bcab83a583499725592ce Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 17:27:05 -0700 Subject: [PATCH 13/14] feedback --- codex-rs/rollout/src/policy.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/codex-rs/rollout/src/policy.rs b/codex-rs/rollout/src/policy.rs index 3ed7dbdd839..c4b4b8c3391 100644 --- a/codex-rs/rollout/src/policy.rs +++ b/codex-rs/rollout/src/policy.rs @@ -11,7 +11,6 @@ pub enum EventPersistenceMode { /// Whether a rollout `item` should be persisted in rollout files for the /// provided persistence `mode`. -#[inline] pub fn is_persisted_response_item(item: &RolloutItem, mode: EventPersistenceMode) -> bool { match item { RolloutItem::ResponseItem(item) => should_persist_response_item(item), From c0d34c87f5713a476855e7eef24afe6a4a1bead9 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 24 Mar 2026 17:28:25 -0700 Subject: [PATCH 14/14] init --- codex-rs/core/src/state_db_bridge.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/codex-rs/core/src/state_db_bridge.rs b/codex-rs/core/src/state_db_bridge.rs index 1465299f393..f073c498b54 100644 --- a/codex-rs/core/src/state_db_bridge.rs +++ b/codex-rs/core/src/state_db_bridge.rs @@ -16,10 +16,6 @@ pub use codex_state::LogEntry; use crate::config::Config; -pub(crate) async fn init(config: &Config) -> Option { - rollout_state_db::init(config).await -} - pub async fn get_state_db(config: &Config) -> Option { rollout_state_db::get_state_db(config).await }