From 0d4bf1c04ac9b301b37117b81cff07cf702f0c37 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Fri, 10 Apr 2026 08:17:25 +0200 Subject: [PATCH 1/4] fix: handle comma-separated coordinates in move_mouse tool --- src/askui/tools/computer/move_mouse_tool.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/askui/tools/computer/move_mouse_tool.py b/src/askui/tools/computer/move_mouse_tool.py index d04db9dd..aa49d8dd 100644 --- a/src/askui/tools/computer/move_mouse_tool.py +++ b/src/askui/tools/computer/move_mouse_tool.py @@ -29,9 +29,16 @@ def __init__(self, agent_os: ComputerAgentOsFacade | None = None) -> None: self.is_cacheable = True def __call__(self, x: int, y: int) -> str: - # for some reason, the agent occasionally calls the tool with the coords - # encoded as strings, which will lead the tool to failing. To prevent this we - # will explicitly convert to int here + # The agent occasionally passes coordinates incorrectly: + # 1. As strings instead of ints (e.g., x="330", y="182") + # 2. Both coords as a single comma-separated string in x + # (e.g., x="330, 182" or x="330, ") + # We handle both cases here. + if isinstance(x, str) and "," in x: + parts = [p.strip() for p in x.split(",") if p.strip()] + x = parts[0] + if len(parts) > 1: + y = parts[1] x, y = int(x), int(y) self.agent_os.mouse_move(x, y) return f"Mouse was moved to position ({x}, {y})." From f9c2204322ba406236a837a16c1ea5af58102792 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Fri, 10 Apr 2026 08:25:27 +0200 Subject: [PATCH 2/4] fix: qa issues --- src/askui/tools/computer/move_mouse_tool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/askui/tools/computer/move_mouse_tool.py b/src/askui/tools/computer/move_mouse_tool.py index aa49d8dd..01505f51 100644 --- a/src/askui/tools/computer/move_mouse_tool.py +++ b/src/askui/tools/computer/move_mouse_tool.py @@ -34,8 +34,8 @@ def __call__(self, x: int, y: int) -> str: # 2. Both coords as a single comma-separated string in x # (e.g., x="330, 182" or x="330, ") # We handle both cases here. - if isinstance(x, str) and "," in x: - parts = [p.strip() for p in x.split(",") if p.strip()] + if isinstance(x, str) and "," in x: # type: ignore[unreachable] + parts = [p.strip() for p in x.split(",") if p.strip()] # type: ignore[unreachable] x = parts[0] if len(parts) > 1: y = parts[1] From 6c3df86b25925dc1541ab32b06bb9823a9a370ab Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Fri, 10 Apr 2026 09:55:41 +0200 Subject: [PATCH 3/4] fix: address review comments --- src/askui/tools/computer/move_mouse_tool.py | 31 ++++++++++++++------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/askui/tools/computer/move_mouse_tool.py b/src/askui/tools/computer/move_mouse_tool.py index 01505f51..ee254a21 100644 --- a/src/askui/tools/computer/move_mouse_tool.py +++ b/src/askui/tools/computer/move_mouse_tool.py @@ -1,3 +1,5 @@ +import re + from askui.models.shared import ComputerBaseTool, ToolTags from askui.tools.computer_agent_os_facade import ComputerAgentOsFacade @@ -8,17 +10,20 @@ class ComputerMoveMouseTool(ComputerBaseTool): def __init__(self, agent_os: ComputerAgentOsFacade | None = None) -> None: super().__init__( name="move_mouse", - description="Move the mouse to a specific position.", + description="""Move the mouse to a specific position. + Pass x and y as separate integer values, not as a combined string.""", input_schema={ "type": "object", "properties": { "x": { "type": "integer", - "description": "The x coordinate of the mouse position as int.", + "description": """The x (horizontal) pixel coordinate. + Must be a single integer, e.g. 330.""", }, "y": { "type": "integer", - "description": "The y coordinate of the mouse position as int.", + "description": """The y (vertical) pixel coordinate. + Must be a single integer, e.g. 182.""", }, }, "required": ["x", "y"], @@ -33,12 +38,18 @@ def __call__(self, x: int, y: int) -> str: # 1. As strings instead of ints (e.g., x="330", y="182") # 2. Both coords as a single comma-separated string in x # (e.g., x="330, 182" or x="330, ") - # We handle both cases here. - if isinstance(x, str) and "," in x: # type: ignore[unreachable] - parts = [p.strip() for p in x.split(",") if p.strip()] # type: ignore[unreachable] - x = parts[0] - if len(parts) > 1: - y = parts[1] - x, y = int(x), int(y) + # We extract all numbers from the string representations to handle both cases. + if not (isinstance(x, int) and isinstance(y, int)): + x, y = self._parse_coordinates(x, y) # type: ignore[unreachable] self.agent_os.mouse_move(x, y) return f"Mouse was moved to position ({x}, {y})." + + @staticmethod + def _parse_coordinates(x: float | str, y: float | str) -> tuple[int, int]: + _NUMBER_PATTERN = re.compile(r"-?\d+") + combined = f"{x},{y}" + numbers = _NUMBER_PATTERN.findall(combined) + if len(numbers) < 2: + error_msg = f"Could not parse coordinates from x={x!r}, y={y!r}" + raise ValueError(error_msg) + return int(numbers[0]), int(numbers[1]) From 333a4ce504060ba33d646e32818441597ededaba Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Fri, 10 Apr 2026 13:51:18 +0200 Subject: [PATCH 4/4] chore: improve error message to give more detailed feedback to agent --- src/askui/tools/computer/move_mouse_tool.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/askui/tools/computer/move_mouse_tool.py b/src/askui/tools/computer/move_mouse_tool.py index ee254a21..2080654c 100644 --- a/src/askui/tools/computer/move_mouse_tool.py +++ b/src/askui/tools/computer/move_mouse_tool.py @@ -49,7 +49,9 @@ def _parse_coordinates(x: float | str, y: float | str) -> tuple[int, int]: _NUMBER_PATTERN = re.compile(r"-?\d+") combined = f"{x},{y}" numbers = _NUMBER_PATTERN.findall(combined) - if len(numbers) < 2: - error_msg = f"Could not parse coordinates from x={x!r}, y={y!r}" + if not len(numbers) == 2: + error_msg = f"""Could not parse mouse_move coordinates from provided + parameters x={x}, y={y}. The parameters x and y must be passed as separate + integer values!""" raise ValueError(error_msg) return int(numbers[0]), int(numbers[1])