From 257b4c93c6418ef0b425ce85432ceeac076038a0 Mon Sep 17 00:00:00 2001 From: bowen628 Date: Wed, 1 Apr 2026 18:07:51 +0800 Subject: [PATCH] feat(agentic): primary model vision gating and computer-use tool updates - Resolve primary model ImageUnderstanding from config; pass into prompt builder, tool context, and API message stripping. - Add ToolOptions / description_with_context for vision-aware tool schemas. - Expand ComputerUse desktop host and tool docs; fix format string delimiter. - Session snapshot and i18n tweaks for session config. --- src/apps/desktop/src/api/config_api.rs | 7 + .../desktop/src/computer_use/desktop_host.rs | 118 +++++++++-- .../core/src/agentic/agents/agentic_mode.rs | 1 + .../core/src/agentic/agents/cowork_mode.rs | 1 + .../core/src/agentic/agents/debug_mode.rs | 1 + .../core/src/agentic/agents/init_agent.rs | 1 + .../core/src/agentic/agents/plan_mode.rs | 1 + .../agents/prompt_builder/prompt_builder.rs | 26 ++- .../core/src/agentic/agents/registry.rs | 12 +- .../src/agentic/execution/execution_engine.rs | 119 +++++++---- .../core/src/agentic/tools/framework.rs | 21 ++ .../tools/implementations/bash_tool.rs | 4 +- .../tools/implementations/code_review_tool.rs | 8 + .../implementations/computer_use_locate.rs | 5 - .../computer_use_mouse_click_tool.rs | 7 +- .../computer_use_mouse_precise_tool.rs | 7 +- .../computer_use_mouse_step_tool.rs | 7 +- .../implementations/computer_use_tool.rs | 197 ++++++++++++++++-- src/crates/core/src/service/config/types.rs | 2 +- .../core/src/service/snapshot/manager.rs | 9 + .../en-US/settings/session-config.json | 4 +- .../zh-CN/settings/session-config.json | 4 +- 22 files changed, 443 insertions(+), 119 deletions(-) diff --git a/src/apps/desktop/src/api/config_api.rs b/src/apps/desktop/src/api/config_api.rs index 1ab92667..23b5ca88 100644 --- a/src/apps/desktop/src/api/config_api.rs +++ b/src/apps/desktop/src/api/config_api.rs @@ -271,6 +271,13 @@ pub async fn get_mode_configs(state: State<'_, AppState>) -> Result f64 { + let t = t.clamp(0.0, 1.0); + t * t * (3.0 - 2.0 * t) + } + + /// Move the pointer along a short visible path instead of warping in one event. + #[cfg(target_os = "macos")] + fn smooth_mouse_move_cg_global(x1: f64, y1: f64) -> BitFunResult<()> { + const MIN_DIST: f64 = 2.5; + const MIN_STEPS: usize = 8; + const MAX_STEPS: usize = 85; + const MAX_DURATION_MS: u64 = 400; + + let (x0, y0) = macos::quartz_mouse_location().unwrap_or((x1, y1)); + let dx = x1 - x0; + let dy = y1 - y0; + let dist = (dx * dx + dy * dy).sqrt(); + if dist < MIN_DIST { + return Self::post_mouse_moved_cg_global(x1, y1); + } + let duration_ms = (70.0 + dist * 0.28).min(MAX_DURATION_MS as f64) as u64; + let steps = ((dist / 5.5).ceil() as usize).clamp(MIN_STEPS, MAX_STEPS); + let step_delay = Duration::from_millis((duration_ms / steps as u64).max(1)); + + for i in 1..=steps { + let t = i as f64 / steps as f64; + let te = Self::smoothstep01(t); + let x = x0 + dx * te; + let y = y0 + dy * te; + Self::post_mouse_moved_cg_global(x, y)?; + if i < steps { + std::thread::sleep(step_delay); + } + } + Ok(()) + } + + /// Windows/Linux: same smooth path using enigo absolute moves (single `Enigo` session). + #[cfg(not(target_os = "macos"))] + fn smooth_mouse_move_enigo_abs(x1: f64, y1: f64) -> BitFunResult<()> { + const MIN_DIST: f64 = 2.5; + const MIN_STEPS: usize = 8; + const MAX_STEPS: usize = 85; + const MAX_DURATION_MS: u64 = 400; + + Self::run_enigo_job(|e| { + let (cx, cy) = e.location().map_err(|err| { + BitFunError::tool(format!("smooth_mouse_move: pointer location: {}", err)) + })?; + let x0 = cx as f64; + let y0 = cy as f64; + let dx = x1 - x0; + let dy = y1 - y0; + let dist = (dx * dx + dy * dy).sqrt(); + if dist < MIN_DIST { + return e + .move_mouse(x1.round() as i32, y1.round() as i32, Coordinate::Abs) + .map_err(|err| BitFunError::tool(format!("mouse_move: {}", err))); + } + let duration_ms = (70.0 + dist * 0.28).min(MAX_DURATION_MS as f64) as u64; + let steps = ((dist / 5.5).ceil() as usize).clamp(MIN_STEPS, MAX_STEPS); + let step_delay = Duration::from_millis((duration_ms / steps as u64).max(1)); + + for i in 1..=steps { + let t = i as f64 / steps as f64; + let te = Self::smoothstep01(t); + let x = x0 + dx * te; + let y = y0 + dy * te; + e.move_mouse(x.round() as i32, y.round() as i32, Coordinate::Abs) + .map_err(|err| BitFunError::tool(format!("mouse_move: {}", err)))?; + if i < steps { + std::thread::sleep(step_delay); + } + } + Ok(()) + }) + } + fn map_button(s: &str) -> BitFunResult