UnitreeSpeak skill over webrtc, Voice Interface added on localhost, Voice interface on mobile device on network

assets/agent/prompt.txt

@@ -64,7 +64,13 @@ Saved Robot Locations:
 
 ***ALWAYS CHECK FIRST if you can find a navigation query in the Saved Robot Locations before running the NavigateWithText tool call. If a saved location is found, get there with NavigateToGoal.***
 
-***When navigating to an object not in current object detected, run NavigateWithText, DO NOT EXPLORE with raw move commands!!!
+***Don't use object detections for navigating to an object, ALWAYS run NavigateWithText. Only use object detections if NavigateWithText fails***
+
+***When running NavigateWithText, set skip_visual_search flag to TRUE if the query is a general location such as kitchen or office, if it fails, then run without this flag***
+
+***When navigating to an object not in current object detected, run NavigateWithText, DO NOT EXPLORE with raw move commands!!!***
+
+***The object detection list is not a comprehensive source of information, when given a visual query like "go to the person wearing a hat" or "Do you see a dog", always Prioritize running observe skill and NavigateWithText***
 
 PLANNING & REASONING:
 - You can develop both short-term and long-term plans to achieve complex goals

dimos/agents/claude_agent.py

@@ -624,8 +624,11 @@ def _observable_query(
             observer.on_completed()
         except Exception as e:
             logger.error(f"Query failed in {self.dev_name}: {e}")
-            observer.on_error(e)
-            self.response_subject.on_error(e)
+            # Send a user-friendly error message instead of propagating the error
+            error_message = "I apologize, but I'm having trouble processing your request right now. Please try again."
+            observer.on_next(error_message)
+            self.response_subject.on_next(error_message)
+            observer.on_completed()
 
     def _handle_tooling(self, response_message, messages):
         """Executes tools and appends tool-use/result blocks to messages."""
@@ -649,20 +652,44 @@ def _handle_tooling(self, response_message, messages):
             }
             messages.append({"role": "assistant", "content": [tool_use_block]})
 
-            # Execute the tool
-            args = json.loads(tool_call.function.arguments)
-            tool_result = self.skills.call(tool_call.function.name, **args)
-
-            # Add tool result to conversation history
-            if tool_result:
+            try:
+                # Execute the tool
+                args = json.loads(tool_call.function.arguments)
+                tool_result = self.skills.call(tool_call.function.name, **args)
+
+                # Check if the result is an error message
+                if isinstance(tool_result, str) and (
+                    "Error executing skill" in tool_result or "is not available" in tool_result
+                ):
+                    # Log the error but provide a user-friendly message
+                    logger.error(f"Tool execution failed: {tool_result}")
+                    tool_result = "I apologize, but I'm having trouble executing that action right now. Please try again or ask for something else."
+
+                # Add tool result to conversation history
+                if tool_result:
+                    messages.append(
+                        {
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "tool_result",
+                                    "tool_use_id": tool_call.id,
+                                    "content": f"{tool_result}",
+                                }
+                            ],
+                        }
+                    )
+            except Exception as e:
+                logger.error(f"Unexpected error executing tool {tool_call.function.name}: {e}")
+                # Add error result to conversation history
                 messages.append(
                     {
                         "role": "user",
                         "content": [
                             {
                                 "type": "tool_result",
                                 "tool_use_id": tool_call.id,
-                                "content": f"{tool_result}",
+                                "content": "I apologize, but I encountered an error while trying to execute that action. Please try again.",
                             }
                         ],
                     }
@@ -672,13 +699,19 @@ def _tooling_callback(self, response_message):
         """Runs the observable query for each tool call in the current response_message"""
         if not hasattr(response_message, "tool_calls") or not response_message.tool_calls:
             return
-        for tool_call in response_message.tool_calls:
-            tool_name = tool_call.function.name
-            tool_id = tool_call.id
-            self.run_observable_query(
-                query_text=f"Tool {tool_name}, ID: {tool_id} execution complete. Please summarize the results and continue.",
-                thinking_budget_tokens=0,
-            ).run()
+
+        try:
+            for tool_call in response_message.tool_calls:
+                tool_name = tool_call.function.name
+                tool_id = tool_call.id
+                self.run_observable_query(
+                    query_text=f"Tool {tool_name}, ID: {tool_id} execution complete. Please summarize the results and continue.",
+                    thinking_budget_tokens=0,
+                ).run()
+        except Exception as e:
+            logger.error(f"Error in tooling callback: {e}")
+            # Continue processing even if the callback fails
+            pass
 
     def _debug_api_call(self, claude_params: dict):
         """Debugging function to log API calls with truncated base64 data."""

dimos/models/qwen/video_query.py

@@ -128,7 +128,7 @@ def query_single_frame(
         openai_client=qwen_client,
         model_name=model_name,
         tokenizer=HuggingFaceTokenizer(model_name=f"Qwen/{model_name}"),
-        max_output_tokens_per_request=100,
+        max_output_tokens_per_request=8192,
         system_query=query,
         pool_scheduler=get_scheduler(),
     )

dimos/perception/object_detection_stream.py

@@ -205,7 +205,7 @@ def process_frame(frame):
                             # position and rotation are already Vector objects, no need to convert
                             robot_pose = self.get_pose()
                             position, rotation = transform_robot_to_map(
-                                robot_pose, position, rotation
+                                robot_pose["position"], robot_pose["rotation"], position, rotation
                             )
                     except Exception as e:
                         logger.error(f"Error transforming to map frame: {e}")

dimos/robot/frontier_exploration/wavefront_frontier_goal_selector.py

@@ -83,7 +83,7 @@ def __init__(
         self,
         min_frontier_size: int = 10,
         occupancy_threshold: int = 65,
-        subsample_resolution: int = 3,
+        subsample_resolution: int = 2,
         min_distance_from_robot: float = 0.5,
         explored_area_buffer: float = 0.5,
         min_distance_from_obstacles: float = 0.6,

dimos/robot/unitree/unitree_skills.py

@@ -51,34 +51,34 @@
         1006,
         "Recovers the robot to a state from which it can take more commands. Useful to run after multiple dynamic commands like front flips.",
     ),
-    (
-        "Euler",
-        1007,
-        "Adjusts the robot's orientation using Euler angles, providing precise control over its rotation.",
-    ),
+    # (
+    #     "Euler",
+    #     1007,
+    #     "Adjusts the robot's orientation using Euler angles, providing precise control over its rotation.",
+    # ),
     # ("Move", 1008, "Move the robot using velocity commands."),  # Intentionally omitted
     ("Sit", 1009, "Commands the robot to sit down from a standing or moving stance."),
-    (
-        "RiseSit",
-        1010,
-        "Commands the robot to rise back to a standing position from a sitting posture.",
-    ),
-    (
-        "SwitchGait",
-        1011,
-        "Switches the robot's walking pattern or style dynamically, suitable for different terrains or speeds.",
-    ),
-    ("Trigger", 1012, "Triggers a specific action or custom routine programmed into the robot."),
-    (
-        "BodyHeight",
-        1013,
-        "Adjusts the height of the robot's body from the ground, useful for navigating various obstacles.",
-    ),
-    (
-        "FootRaiseHeight",
-        1014,
-        "Controls how high the robot lifts its feet during movement, which can be adjusted for different surfaces.",
-    ),
+    # (
+    #     "RiseSit",
+    #     1010,
+    #     "Commands the robot to rise back to a standing position from a sitting posture.",
+    # ),
+    # (
+    #     "SwitchGait",
+    #     1011,
+    #     "Switches the robot's walking pattern or style dynamically, suitable for different terrains or speeds.",
+    # ),
+    # ("Trigger", 1012, "Triggers a specific action or custom routine programmed into the robot."),
+    # (
+    #     "BodyHeight",
+    #     1013,
+    #     "Adjusts the height of the robot's body from the ground, useful for navigating various obstacles.",
+    # ),
+    # (
+    #     "FootRaiseHeight",
+    #     1014,
+    #     "Controls how high the robot lifts its feet during movement, which can be adjusted for different surfaces.",
+    # ),
     (
         "SpeedLevel",
         1015,
@@ -90,16 +90,16 @@
         "Performs a greeting action, which could involve a wave or other friendly gesture.",
     ),
     ("Stretch", 1017, "Engages the robot in a stretching routine."),
-    (
-        "TrajectoryFollow",
-        1018,
-        "Directs the robot to follow a predefined trajectory, which could involve complex paths or maneuvers.",
-    ),
-    (
-        "ContinuousGait",
-        1019,
-        "Enables a mode for continuous walking or running, ideal for long-distance travel.",
-    ),
+    # (
+    #     "TrajectoryFollow",
+    #     1018,
+    #     "Directs the robot to follow a predefined trajectory, which could involve complex paths or maneuvers.",
+    # ),
+    # (
+    #     "ContinuousGait",
+    #     1019,
+    #     "Enables a mode for continuous walking or running, ideal for long-distance travel.",
+    # ),
     ("Content", 1020, "To display or trigger when the robot is happy."),
     ("Wallow", 1021, "The robot falls onto its back and rolls around."),
     (
@@ -108,18 +108,18 @@
         "Performs a predefined dance routine 1, programmed for entertainment or demonstration.",
     ),
     ("Dance2", 1023, "Performs another variant of a predefined dance routine 2."),
-    ("GetBodyHeight", 1024, "Retrieves the current height of the robot's body from the ground."),
-    (
-        "GetFootRaiseHeight",
-        1025,
-        "Retrieves the current height at which the robot's feet are being raised during movement.",
-    ),
-    ("GetSpeedLevel", 1026, "Returns the current speed level at which the robot is operating."),
-    (
-        "SwitchJoystick",
-        1027,
-        "Toggles the control mode to joystick input, allowing for manual direction of the robot's movements.",
-    ),
+    # ("GetBodyHeight", 1024, "Retrieves the current height of the robot's body from the ground."),
+    # (
+    #     "GetFootRaiseHeight",
+    #     1025,
+    #     "Retrieves the current height at which the robot's feet are being raised during movement.",
+    # ),
+    # ("GetSpeedLevel", 1026, "Returns the current speed level at which the robot is operating."),
+    # (
+    #     "SwitchJoystick",
+    #     1027,
+    #     "Toggles the control mode to joystick input, allowing for manual direction of the robot's movements.",
+    # ),
     (
         "Pose",
         1028,
@@ -137,46 +137,46 @@
         1032,
         "Initiates a pouncing movement forward, mimicking animal-like pouncing behavior.",
     ),
-    ("WiggleHips", 1033, "Causes the robot to wiggle its hips."),
-    (
-        "GetState",
-        1034,
-        "Retrieves the current operational state of the robot, including status reports or diagnostic information.",
-    ),
-    (
-        "EconomicGait",
-        1035,
-        "Engages a more energy-efficient walking or running mode to conserve battery life.",
-    ),
-    ("FingerHeart", 1036, "Performs a finger heart gesture while on its hind legs."),
-    (
-        "Handstand",
-        1301,
-        "Commands the robot to perform a handstand, demonstrating balance and control.",
-    ),
-    (
-        "CrossStep",
-        1302,
-        "Engages the robot in a cross-stepping routine, useful for complex locomotion or dance moves.",
-    ),
-    (
-        "OnesidedStep",
-        1303,
-        "Commands the robot to perform a stepping motion that predominantly uses one side.",
-    ),
-    (
-        "Bound",
-        1304,
-        "Initiates a bounding motion, similar to a light, repetitive hopping or leaping.",
-    ),
-    (
-        "LeadFollow",
-        1045,
-        "Engages follow-the-leader behavior, where the robot follows a designated leader or follows a signal.",
-    ),
-    ("LeftFlip", 1042, "Executes a flip towards the left side."),
-    ("RightFlip", 1043, "Performs a flip towards the right side."),
-    ("Backflip", 1044, "Executes a backflip, a complex and dynamic maneuver."),
+    # ("WiggleHips", 1033, "Causes the robot to wiggle its hips."),
+    # (
+    #     "GetState",
+    #     1034,
+    #     "Retrieves the current operational state of the robot, including status reports or diagnostic information.",
+    # ),
+    # (
+    #     "EconomicGait",
+    #     1035,
+    #     "Engages a more energy-efficient walking or running mode to conserve battery life.",
+    # ),
+    # ("FingerHeart", 1036, "Performs a finger heart gesture while on its hind legs."),
+    # (
+    #     "Handstand",
+    #     1301,
+    #     "Commands the robot to perform a handstand, demonstrating balance and control.",
+    # ),
+    # (
+    #     "CrossStep",
+    #     1302,
+    #     "Engages the robot in a cross-stepping routine, useful for complex locomotion or dance moves.",
+    # ),
+    # (
+    #     "OnesidedStep",
+    #     1303,
+    #     "Commands the robot to perform a stepping motion that predominantly uses one side.",
+    # ),
+    # (
+    #     "Bound",
+    #     1304,
+    #     "Initiates a bounding motion, similar to a light, repetitive hopping or leaping.",
+    # ),
+    # (
+    #     "LeadFollow",
+    #     1045,
+    #     "Engages follow-the-leader behavior, where the robot follows a designated leader or follows a signal.",
+    # ),
+    # ("LeftFlip", 1042, "Executes a flip towards the left side."),
+    # ("RightFlip", 1043, "Performs a flip towards the right side."),
+    # ("Backflip", 1044, "Executes a backflip, a complex and dynamic maneuver."),
 ]
 
 # region MyUnitreeSkills

dimos/robot/unitree_webrtc/unitree_skills.py

@@ -50,14 +50,8 @@
     (
         "RecoveryStand",
         1006,
-        "Recovers the robot to a state from which it can take more commands. Useful to run after multiple dynamic commands like front flips.",
+        "Recovers the robot to a state from which it can take more commands. Useful to run after multiple dynamic commands like front flips, Must run after skills like sit and jump and standup.",
     ),
-    (
-        "Euler",
-        1007,
-        "Adjusts the robot's orientation using Euler angles, providing precise control over its rotation.",
-    ),
-    # ("Move", 1008, "Move the robot using velocity commands."),  # Handled separately
     ("Sit", 1009, "Commands the robot to sit down from a standing or moving stance."),
     (
         "RiseSit",