Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions .github/workflows/preview-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,22 @@ jobs:
- name: Install Rust toolchain
shell: bash
run: |
TOOLCHAIN=1.90.0
rustup set profile minimal
rustup toolchain install 1.89.0 --profile minimal --target ${{ matrix.target }}
rustup default 1.89.0
rustup toolchain install "$TOOLCHAIN" --profile minimal
rustup default "$TOOLCHAIN"

if [[ "$RUNNER_OS" == "Linux" ]]; then
rustup target add x86_64-unknown-linux-musl aarch64-unknown-linux-musl
fi

rustup target add "${{ matrix.target }}"

- name: Rust cache (target + registries)
uses: Swatinem/rust-cache@v2
with:
prefix-key: v1-preview
shared-key: preview-${{ matrix.target }}-rust-1.89
shared-key: preview-${{ matrix.target }}-rust-1.90
workspaces: |
code-rs -> target
codex-rs -> target
Expand Down
145 changes: 0 additions & 145 deletions REPORT_issues.md

This file was deleted.

92 changes: 91 additions & 1 deletion code-rs/core/src/chat_completions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ use tokio::sync::mpsc;
use tokio::time::timeout;
use tracing::debug;
use tracing::trace;
use tracing::warn;

use crate::auth::AuthManager;
use crate::ModelProviderInfo;
Expand All @@ -37,6 +38,84 @@ use code_protocol::models::ContentItem;
use code_protocol::models::ReasoningItemContent;
use code_protocol::models::ResponseItem;

/// Sanitizes streamed tool-call arguments by stripping markdown fences and extracting
/// the first valid JSON object or array if possible. Falls back to the trimmed input
/// when no valid JSON can be located.
pub fn sanitize_tool_call_arguments(raw: &str) -> String {
let trimmed = raw.trim();

if trimmed.is_empty() {
return String::new();
}

if serde_json::from_str::<serde_json::Value>(trimmed).is_ok() {
return trimmed.to_string();
}

let without_fences = if let Some(start) = trimmed.find("```") {
let after_start = &trimmed[start + 3..];
let content_start = if let Some(newline_pos) = after_start.find('\n') {
start + 3 + newline_pos + 1
} else {
start + 3
};

if let Some(end) = trimmed[content_start..].find("```") {
&trimmed[content_start..content_start + end]
} else {
&trimmed[content_start..]
}
} else {
trimmed
};

let cleaned = without_fences.trim();

for (idx, ch) in cleaned.char_indices() {
if ch == '{' || ch == '[' {
let candidate = &cleaned[idx..];
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(candidate) {
if let Ok(serialized) = serde_json::to_string(&parsed) {
return serialized;
}
}

let closing = if ch == '{' { '}' } else { ']' };
let mut depth = 0;
let mut in_string = false;
let mut escape_next = false;

for (end_idx, end_ch) in cleaned[idx..].char_indices() {
if escape_next {
escape_next = false;
continue;
}

match end_ch {
'\\' if in_string => escape_next = true,
'"' => in_string = !in_string,
c if c == ch && !in_string => depth += 1,
c if c == closing && !in_string => {
depth -= 1;
if depth == 0 {
let candidate = &cleaned[idx..idx + end_idx + 1];
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(candidate) {
if let Ok(serialized) = serde_json::to_string(&parsed) {
return serialized;
}
}
break;
}
}
_ => {}
}
}
}
}

cleaned.to_string()
}

/// Implementation for the classic Chat Completions API.
pub(crate) async fn stream_chat_completions(
prompt: &Prompt,
Expand Down Expand Up @@ -790,11 +869,22 @@ async fn process_chat_sse<S>(
let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone { item, sequence_number: None, output_index: None })).await;
}

let raw_arguments = fn_call_state.arguments.clone();
let sanitized_arguments = sanitize_tool_call_arguments(&raw_arguments);
if sanitized_arguments != raw_arguments {
warn!(
"Sanitized tool-call arguments for function '{}'. original_len={} sanitized_len={}",
fn_call_state.name.as_deref().unwrap_or(""),
raw_arguments.len(),
sanitized_arguments.len()
);
}

// Then emit the FunctionCall response item.
let item = ResponseItem::FunctionCall {
id: current_item_id.clone(),
name: fn_call_state.name.clone().unwrap_or_else(|| "".to_string()),
arguments: fn_call_state.arguments.clone(),
arguments: sanitized_arguments,
call_id: fn_call_state.call_id.clone().unwrap_or_else(String::new),
};

Expand Down
1 change: 1 addition & 0 deletions code-rs/core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ mod user_notification;
pub mod util;

pub use apply_patch::CODEX_APPLY_PATCH_ARG1;
pub use chat_completions::sanitize_tool_call_arguments;
pub use command_safety::is_safe_command;
pub use safety::get_platform_sandbox;
pub use housekeeping::run_housekeeping_if_due;
Expand Down
58 changes: 58 additions & 0 deletions code-rs/core/tests/tool_call_arguments_sanitizer.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
use code_core::sanitize_tool_call_arguments;
use serde_json::Value;

#[test]
fn valid_object_left_intact() {
let input = r#"{"key": "value"}"#;
let sanitized = sanitize_tool_call_arguments(input);
assert_eq!(sanitized, input);
}

#[test]
fn strips_markdown_fence_with_language() {
let input = "```json\n{\n \"answer\": 42\n}\n```";
let sanitized = sanitize_tool_call_arguments(input);
let value: Value = serde_json::from_str(&sanitized).unwrap();
assert_eq!(value["answer"], 42);
}

#[test]
fn extracts_json_after_prose() {
let input = "Here you go:\n[{'oops': 'not json'}]{\"name\":\"ok\"}";
// Replace single quotes to keep it invalid except for the object portion.
let input = input.replace("'", "\"");
let sanitized = sanitize_tool_call_arguments(&input);
let value: Value = serde_json::from_str(&sanitized).unwrap();
assert_eq!(value["name"], "ok");
}

#[test]
fn handles_arrays() {
let input = "```\n[ {\"tool\": 1}, {\"tool\": 2} ]\n```";
let sanitized = sanitize_tool_call_arguments(input);
let value: Value = serde_json::from_str(&sanitized).unwrap();
assert!(value.is_array());
assert_eq!(value.as_array().unwrap().len(), 2);
}

#[test]
fn preserves_braces_inside_strings() {
let input = r#"{"msg": "brace } inside string"}"#;
let sanitized = sanitize_tool_call_arguments(input);
let value: Value = serde_json::from_str(&sanitized).unwrap();
assert_eq!(value["msg"], "brace } inside string");
}

#[test]
fn returns_trimmed_on_failure() {
let input = "```json\n{ broken json\n```";
let sanitized = sanitize_tool_call_arguments(input);
assert!(!sanitized.contains("```"));
assert!(!sanitized.is_empty());
}

#[test]
fn whitespace_only_yields_empty() {
let sanitized = sanitize_tool_call_arguments(" \n\t");
assert!(sanitized.is_empty());
}
Loading
Loading