diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..6644370b86 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,27 @@ +# top-most EditorConfig file +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +indent_size = 4 + +[*.nix] +indent_size = 2 + +[*.{py,ipynb}] +indent_size = 4 +max_line_length = 100 + +[*.rs] +indent_style = space +indent_size = 4 + +[*.{ts,svelte}] +indent_size = 2 diff --git a/.envrc b/.envrc index 09e580571a..a73ae4a035 100644 --- a/.envrc +++ b/.envrc @@ -2,4 +2,4 @@ if ! has nix_direnv_version || ! nix_direnv_version 3.0.6; then source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.6/direnvrc" "sha256-RYcUJaRMf8oF5LznDrlCXbkOQrywm0HDv1VjYGaJGdM=" fi use flake . -dotenv \ No newline at end of file +dotenv diff --git a/.github/workflows/_docker-build-template.yml b/.github/workflows/_docker-build-template.yml index 730f4a4696..478a9bec84 100644 --- a/.github/workflows/_docker-build-template.yml +++ b/.github/workflows/_docker-build-template.yml @@ -38,19 +38,19 @@ jobs: sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/share/boost sudo rm -rf /usr/local/lib/android - + echo "=== Cleaning images from deleted branches ===" - + # Get list of all remote branches git ls-remote --heads origin | awk '{print $2}' | sed 's|refs/heads/||' > /tmp/active_branches.txt - + # Check each docker image tag against branch list docker images --format "{{.Repository}}:{{.Tag}}|{{.ID}}" | \ grep "ghcr.io/dimensionalos" | \ grep -v ":" | \ while IFS='|' read image_ref id; do tag=$(echo "$image_ref" | cut -d: -f2) - + # Skip if tag matches an active branch if grep -qx "$tag" /tmp/active_branches.txt; then echo "Branch exists: $tag - keeping $image_ref" @@ -59,15 +59,15 @@ jobs: docker rmi "$id" 2>/dev/null || true fi done - + rm -f /tmp/active_branches.txt - + USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') echo "Pre-docker-cleanup disk usage: ${USAGE}%" - + if [ $USAGE -gt 60 ]; then echo "=== Running quick cleanup (usage > 60%) ===" - + # Keep newest image per tag docker images --format "{{.Repository}}|{{.Tag}}|{{.ID}}" | \ grep "ghcr.io/dimensionalos" | \ @@ -81,10 +81,10 @@ jobs: { repo=$1; tag=$2; id=$3 repo_tag = repo ":" tag - + # Skip protected tags if (tag ~ /^(main|dev|latest)$/) next - + # Keep newest per tag, remove older duplicates if (!(repo_tag in seen_combos)) { seen_combos[repo_tag] = 1 @@ -92,28 +92,28 @@ jobs: system("docker rmi " id " 2>/dev/null || true") } }' - + docker image prune -f docker volume prune -f fi - + # Aggressive cleanup if still above 85% USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') if [ $USAGE -gt 85 ]; then echo "=== AGGRESSIVE cleanup (usage > 85%) - removing all except main/dev ===" - + # Remove ALL images except main and dev tags docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}" | \ grep -E "ghcr.io/dimensionalos" | \ grep -vE ":(main|dev)$" | \ awk '{print $2}' | xargs -r docker rmi -f || true - + docker container prune -f docker volume prune -a -f docker network prune -f - docker image prune -f + docker image prune -f fi - + echo -e "post cleanup space:\n $(df -h)" - uses: docker/login-action@v3 diff --git a/.github/workflows/code-cleanup.yml b/.github/workflows/code-cleanup.yml index ddb75a90e3..59cf08dd04 100644 --- a/.github/workflows/code-cleanup.yml +++ b/.github/workflows/code-cleanup.yml @@ -13,20 +13,20 @@ jobs: - name: Fix permissions run: | sudo chown -R $USER:$USER ${{ github.workspace }} || true - + - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - name: Run pre-commit id: pre-commit-first uses: pre-commit/action@v3.0.1 continue-on-error: true - + - name: Re-run pre-commit if failed initially id: pre-commit-retry if: steps.pre-commit-first.outcome == 'failure' uses: pre-commit/action@v3.0.1 continue-on-error: false - + - name: Commit code changes uses: stefanzweifel/git-auto-commit-action@v5 with: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0c6abff68d..dc3aa40a7c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,7 +24,7 @@ jobs: - name: Fix permissions run: | sudo chown -R $USER:$USER ${{ github.workspace }} || true - + - uses: actions/checkout@v4 - id: filter uses: dorny/paths-filter@v3 @@ -132,7 +132,7 @@ jobs: uses: ./.github/workflows/_docker-build-template.yml with: should-run: ${{ - needs.check-changes.result == 'success' && + needs.check-changes.result == 'success' && (needs.check-changes.outputs.dev == 'true' || (needs.ros-python.result == 'success' && (needs.check-changes.outputs.python == 'true' || needs.check-changes.outputs.ros == 'true'))) }} @@ -218,21 +218,20 @@ jobs: # - name: Fix permissions # run: | # sudo chown -R $USER:$USER ${{ github.workspace }} || true - # + # # - uses: actions/checkout@v4 # with: # lfs: true - # + # # - name: Configure Git LFS # run: | # git config --global --add safe.directory '*' # git lfs install # git lfs fetch # git lfs checkout - # + # # - name: Run module tests # env: # CI: "true" # run: | # /entrypoint.sh bash -c "pytest -m module" - diff --git a/.github/workflows/readme.md b/.github/workflows/readme.md index 0bc86973d8..f82ba479bb 100644 --- a/.github/workflows/readme.md +++ b/.github/workflows/readme.md @@ -12,17 +12,17 @@ https://code.visualstudio.com/docs/devcontainers/containers create personal access token (classic, not fine grained) https://github.com/settings/tokens -add permissions +add permissions - read:packages scope to download container images and read their metadata. - + and optionally, - + - write:packages scope to download and upload container images and read and write their metadata. - delete:packages scope to delete container images. more info @ https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry -login to docker via +login to docker via `sh echo TOKEN | docker login ghcr.io -u GITHUB_USER --password-stdin @@ -38,14 +38,14 @@ pull dev image (master) docker pull ghcr.io/dimensionalos/dev:latest ` -# todo +# todo Currently there is an issue with ensuring both correct docker image build ordering, and skipping unneccessary re-builds. (we need job dependancies for builds to wait to their images underneath to be built (for example py waits for ros)) by default if a parent is skipped, it's children get skipped as well, unless they have always() in their conditional. -Issue is once we put always() in the conditional, it seems that no matter what other check we put in the same conditional, job will always run. +Issue is once we put always() in the conditional, it seems that no matter what other check we put in the same conditional, job will always run. for this reason we cannot skip python (and above) builds for now. Needs review. I think we will need to write our own build dispatcher in python that calls github workflows that build images. diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a94839a505..e84d7d43d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,14 +28,14 @@ jobs: # if: ${{ !inputs.should-run }} # run: | # exit 0 - + # - name: Free disk space # run: | # sudo rm -rf /opt/ghc # sudo rm -rf /usr/share/dotnet # sudo rm -rf /usr/local/share/boost # sudo rm -rf /usr/local/lib/android - + run-tests: runs-on: [self-hosted, Linux] container: @@ -47,17 +47,16 @@ jobs: steps: - uses: actions/checkout@v4 - + - name: Fix permissions run: | git config --global --add safe.directory '*' - + - name: Run tests run: | /entrypoint.sh bash -c "${{ inputs.cmd }}" - + - name: check disk space if: failure() run: | df -h - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e38877c44..40b1877835 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,9 @@ repos: - id: trailing-whitespace language: python types: [text] - stages: [pre-push] + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] - id: check-json - id: check-toml - id: check-yaml @@ -41,6 +43,13 @@ repos: name: format json args: [ --autofix, --no-sort-keys ] + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python + rev: 3.4.1 + hooks: + - id: editorconfig-checker + alias: ec + args: [-disable-max-line-length, -disable-indentation] + # - repo: local # hooks: # - id: mypy @@ -60,5 +69,3 @@ repos: pass_filenames: false entry: bin/lfs_check language: script - - diff --git a/.python-version b/.python-version new file mode 100644 index 0000000000..e4fba21835 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/.style.yapf b/.style.yapf index e1d4f5f627..b8d6fb374a 100644 --- a/.style.yapf +++ b/.style.yapf @@ -1,3 +1,3 @@ [style] based_on_style = google - column_limit = 80 \ No newline at end of file + column_limit = 80 diff --git a/AUTONOMY_STACK_README.md b/AUTONOMY_STACK_README.md index 70eff131ce..4a74500b2f 100644 --- a/AUTONOMY_STACK_README.md +++ b/AUTONOMY_STACK_README.md @@ -122,7 +122,7 @@ cd ~/autonomy_stack_mecanum_wheel_platform | `/goal_pose` | `geometry_msgs/PoseStamped` | Send goal with orientation | | `/cancel_goal` | `std_msgs/Bool` | Cancel current goal (data: true) | | `/joy` | `sensor_msgs/Joy` | Joystick input | -| `/stop` | `std_msgs/Int8` | Soft Stop (2=stop all commmand, 0 = release) | +| `/stop` | `std_msgs/Int8` | Soft Stop (2=stop all commmand, 0 = release) | | `/navigation_boundary` | `geometry_msgs/PolygonStamped` | Set navigation boundaries | | `/added_obstacles` | `sensor_msgs/PointCloud2` | Virtual obstacles | @@ -281,4 +281,4 @@ init_yaw: 0.0 mapping_line_resolution: 0.1 # Decrease for higher quality mapping_plane_resolution: 0.2 # Decrease for higher quality max_iterations: 5 # Increase for better accuracy -``` \ No newline at end of file +``` diff --git a/LICENSE b/LICENSE index b06471524c..5e2927e3ad 100644 --- a/LICENSE +++ b/LICENSE @@ -14,4 +14,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. diff --git a/README.md b/README.md index 1db93e9887..f8df86f5aa 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,11 @@ ## What is Dimensional? -Dimensional is an open-source framework for building agentive generalist robots. DimOS allows off-the-shelf Agents to call tools/functions and read sensor/state data directly from ROS. +Dimensional is an open-source framework for building agentive generalist robots. DimOS allows off-the-shelf Agents to call tools/functions and read sensor/state data directly from ROS. -The framework enables neurosymbolic orchestration of Agents as generalized spatial reasoners/planners and Robot state/action primitives as functions. +The framework enables neurosymbolic orchestration of Agents as generalized spatial reasoners/planners and Robot state/action primitives as functions. -The result: cross-embodied *"Dimensional Applications"* exceptional at generalization and robust at symbolic action execution. +The result: cross-embodied *"Dimensional Applications"* exceptional at generalization and robust at symbolic action execution. ## DIMOS x Unitree Go2 (OUT OF DATE) @@ -38,18 +38,18 @@ We are shipping a first look at the DIMOS x Unitree Go2 integration, allowing fo - Lidar / PointCloud primitives - Any other generic Unitree ROS2 topics -### Features +### Features - **DimOS Agents** - Agent() classes with planning, spatial reasoning, and Robot.Skill() function calling abilities. - Integrate with any off-the-shelf hosted or local model: OpenAIAgent, ClaudeAgent, GeminiAgent ๐Ÿšง, DeepSeekAgent ๐Ÿšง, HuggingFaceRemoteAgent, HuggingFaceLocalAgent, etc. - - Modular agent architecture for easy extensibility and chaining of Agent output --> Subagents input. + - Modular agent architecture for easy extensibility and chaining of Agent output --> Subagents input. - Agent spatial / language memory for location grounded reasoning and recall. - **DimOS Infrastructure** - A reactive data streaming architecture using RxPY to manage real-time video (or other sensor input), outbound commands, and inbound robot state between the DimOS interface, Agents, and ROS2. - - Robot Command Queue to handle complex multi-step actions to Robot. - - Simulation bindings (Genesis, Isaacsim, etc.) to test your agentive application before deploying to a physical robot. + - Robot Command Queue to handle complex multi-step actions to Robot. + - Simulation bindings (Genesis, Isaacsim, etc.) to test your agentive application before deploying to a physical robot. - **DimOS Interface / Development Tools** - Local development interface to control your robot, orchestrate agents, visualize camera/lidar streams, and debug your dimensional agentive application. @@ -92,27 +92,27 @@ pip install -e .[cuda,dev] cp default.env .env ``` -#### Test the install -```bash +#### Test the install +```bash pytest -s dimos/ ``` #### Test Dimensional with a replay UnitreeGo2 stream (no robot required) ```bash -CONNECTION_TYPE=replay python dimos/robot/unitree_webrtc/unitree_go2.py +CONNECTION_TYPE=replay python dimos/robot/unitree_webrtc/unitree_go2.py ``` #### Test Dimensional with a simulated UnitreeGo2 in MuJoCo (no robot required) ```bash pip install -e .[sim] export DISPLAY=:1 # Or DISPLAY=:0 if getting GLFW/OpenGL X11 errors -CONNECTION_TYPE=mujoco python dimos/robot/unitree_webrtc/unitree_go2.py +CONNECTION_TYPE=mujoco python dimos/robot/unitree_webrtc/unitree_go2.py ``` #### Test Dimensional with a real UnitreeGo2 over WebRTC ```bash export ROBOT_IP=192.168.X.XXX # Add the robot IP address -python dimos/robot/unitree_webrtc/unitree_go2.py +python dimos/robot/unitree_webrtc/unitree_go2.py ``` #### Test Dimensional with a real UnitreeGo2 running Agents @@ -127,7 +127,7 @@ python dimos/robot/unitree_webrtc/run_agents2.py Full functionality will require API keys for the following: -Requirements: +Requirements: - OpenAI API key (required for all LLMAgents due to OpenAIEmbeddings) - Claude API key (required for ClaudeAgent) - Alibaba API key (required for Navigation skills) @@ -139,10 +139,10 @@ export CLAUDE_API_KEY= export ALIBABA_API_KEY= ``` -### ROS2 Unitree Go2 SDK Installation +### ROS2 Unitree Go2 SDK Installation #### System Requirements -- Ubuntu 22.04 +- Ubuntu 22.04 - ROS2 Distros: Iron, Humble, Rolling See [Unitree Go2 ROS2 SDK](https://github.com/dimensionalOS/go2_ros2_sdk) for additional installation instructions. @@ -167,7 +167,7 @@ colcon build ### Run the test application -#### ROS2 Terminal: +#### ROS2 Terminal: ```bash # Change path to your Go2 ROS2 SDK installation source /ros2_ws/install/setup.bash @@ -179,7 +179,7 @@ ros2 launch go2_robot_sdk robot.launch.py ``` -#### Python Terminal: +#### Python Terminal: ```bash # Change path to your Go2 ROS2 SDK installation source /ros2_ws/install/setup.bash @@ -189,7 +189,7 @@ python tests/run.py #### DimOS Interface: ```bash cd dimos/web/dimos_interface -yarn install +yarn install yarn dev # you may need to run sudo if previously built via Docker ``` @@ -224,7 +224,7 @@ yarn dev # you may need to run sudo if previously built via Docker โ”‚ โ”œโ”€โ”€ types/ # Type definitions and interfaces โ”‚ โ”œโ”€โ”€ utils/ # Utility functions and helpers โ”‚ โ””โ”€โ”€ web/ # DimOS development interface and API -โ”‚ โ”œโ”€โ”€ dimos_interface/ # DimOS web interface +โ”‚ โ”œโ”€โ”€ dimos_interface/ # DimOS web interface โ”‚ โ””โ”€โ”€ websocket_vis/ # Websocket visualizations โ”œโ”€โ”€ tests/ # Test files โ”‚ โ”œโ”€โ”€ genesissim/ # Genesis simulator tests @@ -267,11 +267,11 @@ while True: # keep process running ### DimOS Application with Agent chaining (OUT OF DATE) -Let's build a simple DimOS application with Agent chaining. We define a ```planner``` as a ```PlanningAgent``` that takes in user input to devise a complex multi-step plan. This plan is passed step-by-step to an ```executor``` agent that can queue ```AbstractRobotSkill``` commands to the ```ROSCommandQueue```. +Let's build a simple DimOS application with Agent chaining. We define a ```planner``` as a ```PlanningAgent``` that takes in user input to devise a complex multi-step plan. This plan is passed step-by-step to an ```executor``` agent that can queue ```AbstractRobotSkill``` commands to the ```ROSCommandQueue```. -Our reactive Pub/Sub data streaming architecture allows for chaining of ```Agent_0 --> Agent_1 --> ... --> Agent_n``` via the ```input_query_stream``` parameter in each which takes an ```Observable``` input from the previous Agent in the chain. +Our reactive Pub/Sub data streaming architecture allows for chaining of ```Agent_0 --> Agent_1 --> ... --> Agent_n``` via the ```input_query_stream``` parameter in each which takes an ```Observable``` input from the previous Agent in the chain. -**Via this method you can chain together any number of Agents() to create complex dimensional applications.** +**Via this method you can chain together any number of Agents() to create complex dimensional applications.** ```python @@ -371,8 +371,8 @@ First, define your skill. For instance, a `GreeterSkill` that can deliver a conf ```python class GreeterSkill(AbstractSkill): """Greats the user with a friendly message.""" # Gives the agent better context for understanding (the more detailed the better). - - greeting: str = Field(..., description="The greating message to display.") # The field needed for the calling of the function. Your agent will also pull from the description here to gain better context. + + greeting: str = Field(..., description="The greating message to display.") # The field needed for the calling of the function. Your agent will also pull from the description here to gain better context. def __init__(self, greeting_message: Optional[str] = None, **data): super().__init__(**data) @@ -410,19 +410,19 @@ Define the SkillLibrary and any skills it will manage in its collection: ```python class MovementSkillsLibrary(SkillLibrary): """A specialized skill library containing movement and navigation related skills.""" - + def __init__(self, robot=None): super().__init__() self._robot = robot - + def initialize_skills(self, robot=None): """Initialize all movement skills with the robot instance.""" if robot: self._robot = robot - + if not self._robot: raise ValueError("Robot instance is required to initialize skills") - + # Initialize with all movement-related skills self.add(Navigate(robot=self._robot)) self.add(NavigateToGoal(robot=self._robot)) @@ -431,7 +431,7 @@ class MovementSkillsLibrary(SkillLibrary): self.add(GetPose(robot=self._robot)) # Position tracking skill ``` -Note the addision of initialized skills added to this collection above. +Note the addision of initialized skills added to this collection above. Proceed to use this skill library in an Agent: @@ -450,9 +450,9 @@ performing_agent = OpenAIAgent( ``` ### Unitree Test Files -- **`tests/run_go2_ros.py`**: Tests `UnitreeROSControl(ROSControl)` initialization in `UnitreeGo2(Robot)` via direct function calls `robot.move()` and `robot.webrtc_req()` +- **`tests/run_go2_ros.py`**: Tests `UnitreeROSControl(ROSControl)` initialization in `UnitreeGo2(Robot)` via direct function calls `robot.move()` and `robot.webrtc_req()` - **`tests/simple_agent_test.py`**: Tests a simple zero-shot class `OpenAIAgent` example -- **`tests/unitree/test_webrtc_queue.py`**: Tests `ROSCommandQueue` via a 20 back-to-back WebRTC requests to the robot +- **`tests/unitree/test_webrtc_queue.py`**: Tests `ROSCommandQueue` via a 20 back-to-back WebRTC requests to the robot - **`tests/test_planning_agent_web_interface.py`**: Tests a simple two-stage `PlanningAgent` chained to an `ExecutionAgent` with backend FastAPI interface. - **`tests/test_unitree_agent_queries_fastapi.py`**: Tests a zero-shot `ExecutionAgent` with backend FastAPI interface. @@ -470,11 +470,11 @@ This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENS ## Acknowledgments -Huge thanks to! -- The Roboverse Community and their unitree-specific help. Check out their [Discord](https://discord.gg/HEXNMCNhEh). -- @abizovnuralem for his work on the [Unitree Go2 ROS2 SDK](https://github.com/abizovnuralem/go2_ros2_sdk) we integrate with for DimOS. -- @legion1581 for his work on the [Unitree Go2 WebRTC Connect](https://github.com/legion1581/go2_webrtc_connect) from which we've pulled the ```Go2WebRTCConnection``` class and other types for seamless WebRTC-only integration with DimOS. -- @tfoldi for the webrtc_req integration via Unitree Go2 ROS2 SDK, which allows for seamless usage of Unitree WebRTC control primitives with DimOS. +Huge thanks to! +- The Roboverse Community and their unitree-specific help. Check out their [Discord](https://discord.gg/HEXNMCNhEh). +- @abizovnuralem for his work on the [Unitree Go2 ROS2 SDK](https://github.com/abizovnuralem/go2_ros2_sdk) we integrate with for DimOS. +- @legion1581 for his work on the [Unitree Go2 WebRTC Connect](https://github.com/legion1581/go2_webrtc_connect) from which we've pulled the ```Go2WebRTCConnection``` class and other types for seamless WebRTC-only integration with DimOS. +- @tfoldi for the webrtc_req integration via Unitree Go2 ROS2 SDK, which allows for seamless usage of Unitree WebRTC control primitives with DimOS. ## Contact @@ -482,6 +482,5 @@ Huge thanks to! - Email: [build@dimensionalOS.com](mailto:build@dimensionalOS.com) ## Known Issues -- Agent() failure to execute Nav2 action primitives (move, reverse, spinLeft, spinRight) is almost always due to the internal ROS2 collision avoidance, which will sometimes incorrectly display obstacles or be overly sensitive. Look for ```[behavior_server]: Collision Ahead - Exiting DriveOnHeading``` in the ROS logs. Reccomend restarting ROS2 or moving robot from objects to resolve. +- Agent() failure to execute Nav2 action primitives (move, reverse, spinLeft, spinRight) is almost always due to the internal ROS2 collision avoidance, which will sometimes incorrectly display obstacles or be overly sensitive. Look for ```[behavior_server]: Collision Ahead - Exiting DriveOnHeading``` in the ROS logs. Reccomend restarting ROS2 or moving robot from objects to resolve. - ```docker-compose up --build``` does not fully initialize the ROS2 environment due to ```std::bad_alloc``` errors. This will occur during continuous docker development if the ```docker-compose down``` is not run consistently before rebuilding and/or you are on a machine with less RAM, as ROS is very memory intensive. Reccomend running to clear your docker cache/images/containers with ```docker system prune``` and rebuild. - diff --git a/assets/agent/prompt.txt b/assets/agent/prompt.txt index f38c13eb13..346e0388e0 100644 --- a/assets/agent/prompt.txt +++ b/assets/agent/prompt.txt @@ -108,4 +108,4 @@ Example: If a user asks "Can you find the kitchen?", you would: 1. Acknowledge: "I'll help you find the kitchen." 2. Execute: Call the Navigate skill with query="kitchen" 3. Feedback: Report success or failure of navigation attempt -4. Next steps: Offer to take further actions once at the kitchen location \ No newline at end of file +4. Next steps: Offer to take further actions once at the kitchen location diff --git a/assets/dimensionalascii.txt b/assets/dimensionalascii.txt index 3202258acb..9b35fb8778 100644 --- a/assets/dimensionalascii.txt +++ b/assets/dimensionalascii.txt @@ -1,8 +1,7 @@ - โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— - โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ - โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ - โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ•šโ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ + โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— + โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ + โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ + โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ•šโ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ•šโ•โ• โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ•โ•šโ•โ• โ•šโ•โ•โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•โ•โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ• โ•šโ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•šโ•โ•โ•โ•โ•โ•โ• - diff --git a/assets/license_file_header.txt b/assets/license_file_header.txt index 4268cd990f..a02322f92f 100644 --- a/assets/license_file_header.txt +++ b/assets/license_file_header.txt @@ -10,4 +10,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +limitations under the License. diff --git a/base-requirements.txt b/base-requirements.txt index 6d4cb9671c..68b485fb9a 100644 --- a/base-requirements.txt +++ b/base-requirements.txt @@ -1,2 +1,2 @@ torch==2.0.1 -torchvision==0.15.2 \ No newline at end of file +torchvision==0.15.2 diff --git a/bin/dev b/bin/dev index 509fa5e14d..2ccebcb071 100755 --- a/bin/dev +++ b/bin/dev @@ -59,7 +59,7 @@ get_tag() { build_image() { local image_tag image_tag=$(get_tag) - + docker build \ --build-arg GIT_COMMIT=$(git rev-parse --short HEAD) \ --build-arg GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) \ diff --git a/bin/lfs_push b/bin/lfs_push index 68b1326e49..0d9e01d743 100755 --- a/bin/lfs_push +++ b/bin/lfs_push @@ -28,27 +28,27 @@ compressed_dirs=() for dir_path in data/*; do # Skip if no directories found (glob didn't match) [ ! "$dir_path" ] && continue - + # Extract directory name dir_name=$(basename "$dir_path") - + # Skip .lfs directory if it exists [ "$dir_name" = ".lfs" ] && continue - + # Define compressed file path compressed_file="data/.lfs/${dir_name}.tar.gz" - + # Check if compressed file already exists if [ -f "$compressed_file" ]; then continue fi - + echo -e " ${YELLOW}Compressing${NC} $dir_path -> $compressed_file" - + # Show directory size before compression dir_size=$(du -sh "$dir_path" | cut -f1) echo -e " Data size: ${YELLOW}$dir_size${NC}" - + # Create compressed archive with progress bar # Use tar with gzip compression, excluding hidden files and common temp files tar -czf "$compressed_file" \ @@ -60,13 +60,13 @@ for dir_path in data/*; do --checkpoint-action=dot \ -C "data/" \ "$dir_name" - + if [ $? -eq 0 ]; then # Show compressed file size compressed_size=$(du -sh "$compressed_file" | cut -f1) echo -e " ${GREEN}โœ“${NC} Successfully compressed $dir_name (${GREEN}$dir_size${NC} โ†’ ${GREEN}$compressed_size${NC})" compressed_dirs+=("$dir_name") - + # Add the compressed file to git LFS tracking git add -f "$compressed_file" @@ -87,7 +87,7 @@ if [ ${#compressed_dirs[@]} -gt 0 ]; then dirs_list=$(IFS=', '; echo "${compressed_dirs[*]}") commit_msg="Auto-compress test data: ${dirs_list}" fi - + #git commit -m "$commit_msg" echo -e "${GREEN}โœ“${NC} Compressed file references added. Uploading..." git lfs push origin $(git branch --show-current) @@ -95,4 +95,3 @@ if [ ${#compressed_dirs[@]} -gt 0 ]; then else echo -e "${GREEN}โœ“${NC} No test data to compress" fi - diff --git a/bin/robot-debugger b/bin/robot-debugger index d9bef015e7..165a546a0c 100755 --- a/bin/robot-debugger +++ b/bin/robot-debugger @@ -8,7 +8,7 @@ # # And now start this script # -# $ ./bin/robot-debugger +# $ ./bin/robot-debugger # >>> robot.explore() # True # >>> diff --git a/default.env b/default.env index 0c7a0ff14a..5098a60892 100644 --- a/default.env +++ b/default.env @@ -10,6 +10,6 @@ WEBRTC_SERVER_HOST=0.0.0.0 WEBRTC_SERVER_PORT=9991 DISPLAY=:0 -# Optional +# Optional #DIMOS_MAX_WORKERS= TEST_RTSP_URL= diff --git a/dimos/__init__.py b/dimos/__init__.py index 8b13789179..e69de29bb2 100644 --- a/dimos/__init__.py +++ b/dimos/__init__.py @@ -1 +0,0 @@ - diff --git a/dimos/hardware/README.md b/dimos/hardware/README.md index fb598e82cf..2587e3595d 100644 --- a/dimos/hardware/README.md +++ b/dimos/hardware/README.md @@ -26,4 +26,4 @@ On receiver machine: ```bash python3 dimos/hardware/gstreamer_camera_test_script.py --host 10.0.0.227 --port 5000 -``` \ No newline at end of file +``` diff --git a/dimos/hardware/can_activate.sh b/dimos/hardware/can_activate.sh index 60cc95e7ea..addb892557 100644 --- a/dimos/hardware/can_activate.sh +++ b/dimos/hardware/can_activate.sh @@ -37,12 +37,12 @@ if [ "$CURRENT_CAN_COUNT" -ne "1" ]; then for iface in $(ip -br link show type can | awk '{print $1}'); do # Use ethtool to retrieve bus-info. BUS_INFO=$(sudo ethtool -i "$iface" | grep "bus-info" | awk '{print $2}') - + if [ -z "$BUS_INFO" ];then echo "Error: Unable to retrieve bus-info for interface $iface." continue fi - + echo "Interface $iface is inserted into USB port $BUS_INFO" done echo -e " \e[31m Error: The number of CAN modules detected by the system ($CURRENT_CAN_COUNT) does not match the expected number (1). \e[0m" @@ -62,7 +62,7 @@ fi if [ -n "$USB_ADDRESS" ]; then echo "Detected USB hardware address parameter: $USB_ADDRESS" - + # Use ethtool to find the CAN interface corresponding to the USB hardware address. INTERFACE_NAME="" for iface in $(ip -br link show type can | awk '{print $1}'); do @@ -72,7 +72,7 @@ if [ -n "$USB_ADDRESS" ]; then break fi done - + if [ -z "$INTERFACE_NAME" ]; then echo "Error: Unable to find CAN interface corresponding to USB hardware address $USB_ADDRESS." exit 1 @@ -82,7 +82,7 @@ if [ -n "$USB_ADDRESS" ]; then else # Retrieve the unique CAN interface. INTERFACE_NAME=$(ip -br link show type can | awk '{print $1}') - + # Check if the interface name has been retrieved. if [ -z "$INTERFACE_NAME" ]; then echo "Error: Unable to detect CAN interface." @@ -100,7 +100,7 @@ CURRENT_BITRATE=$(ip -details link show "$INTERFACE_NAME" | grep -oP 'bitrate \K if [ "$IS_LINK_UP" = "yes" ] && [ "$CURRENT_BITRATE" -eq "$DEFAULT_BITRATE" ]; then echo "Interface $INTERFACE_NAME is already activated with a bitrate of $DEFAULT_BITRATE." - + # Check if the interface name matches the default name. if [ "$INTERFACE_NAME" != "$DEFAULT_CAN_NAME" ]; then echo "Rename interface $INTERFACE_NAME to $DEFAULT_CAN_NAME." @@ -118,13 +118,13 @@ else else echo "Interface $INTERFACE_NAME is not activated or bitrate is not set." fi - + # Set the interface bitrate and activate it. sudo ip link set "$INTERFACE_NAME" down sudo ip link set "$INTERFACE_NAME" type can bitrate $DEFAULT_BITRATE sudo ip link set "$INTERFACE_NAME" up echo "Interface $INTERFACE_NAME has been reset to bitrate $DEFAULT_BITRATE and activated." - + # Rename the interface to the default name. if [ "$INTERFACE_NAME" != "$DEFAULT_CAN_NAME" ]; then echo "Rename interface $INTERFACE_NAME to $DEFAULT_CAN_NAME." diff --git a/dimos/hardware/piper_description.urdf b/dimos/hardware/piper_description.urdf index 21209b6dbb..c8a5a11ded 100755 --- a/dimos/hardware/piper_description.urdf +++ b/dimos/hardware/piper_description.urdf @@ -366,7 +366,7 @@ filename="package://piper_description/meshes/gripper_base.STL" /> - + diff --git a/dimos/mapping/osm/README.md b/dimos/mapping/osm/README.md index be3d4a3ee2..cb94c0160b 100644 --- a/dimos/mapping/osm/README.md +++ b/dimos/mapping/osm/README.md @@ -28,7 +28,7 @@ You have to update it with your current location and when you stray too far from ```python curr_map = CurrentLocationMap(QwenVlModel()) -# Set your latest position. +# Set your latest position. curr_map.update_position(LatLon(lat=..., lon=...)) # If you want to get back a GPS position of a feature (Qwen gets your current position). diff --git a/dimos/perception/detection/detectors/config/custom_tracker.yaml b/dimos/perception/detection/detectors/config/custom_tracker.yaml index 4386473086..7a6748ebf6 100644 --- a/dimos/perception/detection/detectors/config/custom_tracker.yaml +++ b/dimos/perception/detection/detectors/config/custom_tracker.yaml @@ -18,4 +18,4 @@ gmc_method: sparseOptFlow # method of global motion compensation # ReID model related thresh (not supported yet) proximity_thresh: 0.6 appearance_thresh: 0.35 -with_reid: False \ No newline at end of file +with_reid: False diff --git a/dimos/perception/segmentation/config/custom_tracker.yaml b/dimos/perception/segmentation/config/custom_tracker.yaml index 4386473086..7a6748ebf6 100644 --- a/dimos/perception/segmentation/config/custom_tracker.yaml +++ b/dimos/perception/segmentation/config/custom_tracker.yaml @@ -18,4 +18,4 @@ gmc_method: sparseOptFlow # method of global motion compensation # ReID model related thresh (not supported yet) proximity_thresh: 0.6 appearance_thresh: 0.35 -with_reid: False \ No newline at end of file +with_reid: False diff --git a/dimos/robot/agilex/README.md b/dimos/robot/agilex/README.md index 1e678cae65..5d43fa3c3f 100644 --- a/dimos/robot/agilex/README.md +++ b/dimos/robot/agilex/README.md @@ -21,27 +21,27 @@ from dimos.types.robot_capabilities import RobotCapability class YourRobot: """Your robot implementation.""" - + def __init__(self, robot_capabilities: Optional[List[RobotCapability]] = None): # Core components self.dimos = None self.modules = {} self.skill_library = SkillLibrary() - + # Define capabilities self.capabilities = robot_capabilities or [ RobotCapability.VISION, RobotCapability.MANIPULATION, ] - + async def start(self): """Start the robot modules.""" # Initialize DIMOS with worker count self.dimos = core.start(2) # Number of workers needed - + # Deploy modules # ... (see Module System section) - + def stop(self): """Stop all modules and clean up.""" # Stop modules @@ -96,7 +96,7 @@ self.camera.color_image.transport = core.LCMTransport( Image # Message type ) self.camera.depth_image.transport = core.LCMTransport( - "/camera/depth_image", + "/camera/depth_image", Image ) ``` @@ -142,23 +142,23 @@ def main(): # 1. Create and start robot robot = YourRobot() asyncio.run(robot.start()) - + # 2. Set up skills skills = robot.get_skills() skills.add(YourSkill) skills.create_instance("YourSkill", robot=robot) - + # 3. Set up reactive streams agent_response_subject = rx.subject.Subject() agent_response_stream = agent_response_subject.pipe(ops.share()) - + # 4. Create web interface web_interface = RobotWebInterface( port=5555, text_streams={"agent_responses": agent_response_stream}, audio_subject=rx.subject.Subject() ) - + # 5. Create agent agent = ClaudeAgent( dev_name="your_agent", @@ -167,12 +167,12 @@ def main(): system_query="Your system prompt here", model_name="claude-3-5-haiku-latest" ) - + # 6. Connect agent responses agent.get_response_observable().subscribe( lambda x: agent_response_subject.on_next(x) ) - + # 7. Run interface web_interface.run() ``` @@ -205,51 +205,51 @@ class MyRobot: self.camera = None self.manipulation = None self.skill_library = SkillLibrary() - + self.capabilities = robot_capabilities or [ RobotCapability.VISION, RobotCapability.MANIPULATION, ] - + async def start(self): # Start DIMOS self.dimos = core.start(2) - + # Enable LCM pubsub.lcm.autoconf() - + # Deploy camera self.camera = self.dimos.deploy( CameraModule, camera_id=0, fps=30 ) - + # Configure camera LCM self.camera.color_image.transport = core.LCMTransport("/camera/rgb", Image) self.camera.depth_image.transport = core.LCMTransport("/camera/depth", Image) self.camera.camera_info.transport = core.LCMTransport("/camera/info", CameraInfo) - + # Deploy manipulation self.manipulation = self.dimos.deploy(ManipulationModule) - + # Connect modules self.manipulation.rgb_image.connect(self.camera.color_image) self.manipulation.depth_image.connect(self.camera.depth_image) self.manipulation.camera_info.connect(self.camera.camera_info) - + # Configure manipulation output self.manipulation.viz_image.transport = core.LCMTransport("/viz/output", Image) - + # Start modules self.camera.start() self.manipulation.start() - + await asyncio.sleep(2) # Allow initialization - + def get_skills(self): return self.skill_library - + def stop(self): if self.manipulation: self.manipulation.stop() @@ -279,29 +279,29 @@ def main(): if not os.getenv("ANTHROPIC_API_KEY"): print("Please set ANTHROPIC_API_KEY") return - + # Create robot robot = MyRobot() - + try: # Start robot asyncio.run(robot.start()) - + # Set up skills skills = robot.get_skills() skills.add(BasicSkill) skills.create_instance("BasicSkill", robot=robot) - + # Set up streams agent_response_subject = rx.subject.Subject() agent_response_stream = agent_response_subject.pipe(ops.share()) - + # Create web interface web_interface = RobotWebInterface( port=5555, text_streams={"agent_responses": agent_response_stream} ) - + # Create agent agent = ClaudeAgent( dev_name="my_agent", @@ -309,17 +309,17 @@ def main(): skills=skills, system_query=SYSTEM_PROMPT ) - + # Connect responses agent.get_response_observable().subscribe( lambda x: agent_response_subject.on_next(x) ) - + print("Robot ready at http://localhost:5555") - + # Run web_interface.run() - + finally: robot.stop() @@ -341,7 +341,7 @@ from dimos.skills import Skill, skill class BasicSkill(Skill): def __init__(self, robot): self.robot = robot - + def run(self, action: str): # Implement skill logic return f"Performed: {action}" @@ -368,4 +368,4 @@ class BasicSkill(Skill): 1. **"Module not started"**: Ensure start() is called after deployment 2. **"No data received"**: Check LCM transport configuration 3. **"Connection failed"**: Verify input/output types match -4. **"Cleanup errors"**: Stop modules before closing DIMOS \ No newline at end of file +4. **"Cleanup errors"**: Stop modules before closing DIMOS diff --git a/dimos/robot/agilex/README_CN.md b/dimos/robot/agilex/README_CN.md index 482a09dd6d..909a309ce9 100644 --- a/dimos/robot/agilex/README_CN.md +++ b/dimos/robot/agilex/README_CN.md @@ -21,27 +21,27 @@ from dimos.types.robot_capabilities import RobotCapability class YourRobot: """ๆ‚จ็š„ๆœบๅ™จไบบๅฎž็Žฐใ€‚""" - + def __init__(self, robot_capabilities: Optional[List[RobotCapability]] = None): # ๆ ธๅฟƒ็ป„ไปถ self.dimos = None self.modules = {} self.skill_library = SkillLibrary() - + # ๅฎšไน‰่ƒฝๅŠ› self.capabilities = robot_capabilities or [ RobotCapability.VISION, RobotCapability.MANIPULATION, ] - + async def start(self): """ๅฏๅŠจๆœบๅ™จไบบๆจกๅ—ใ€‚""" # ๅˆๅง‹ๅŒ– DIMOS๏ผŒๆŒ‡ๅฎšๅทฅไฝœ็บฟ็จ‹ๆ•ฐ self.dimos = core.start(2) # ้œ€่ฆ็š„ๅทฅไฝœ็บฟ็จ‹ๆ•ฐ - + # ้ƒจ็ฝฒๆจกๅ— # ... (ๅ‚่งๆจกๅ—็ณป็ปŸ็ซ ่Š‚) - + def stop(self): """ๅœๆญขๆ‰€ๆœ‰ๆจกๅ—ๅนถๆธ…็†่ต„ๆบใ€‚""" # ๅœๆญขๆจกๅ— @@ -96,7 +96,7 @@ self.camera.color_image.transport = core.LCMTransport( Image # ๆถˆๆฏ็ฑปๅž‹ ) self.camera.depth_image.transport = core.LCMTransport( - "/camera/depth_image", + "/camera/depth_image", Image ) ``` @@ -142,23 +142,23 @@ def main(): # 1. ๅˆ›ๅปบๅนถๅฏๅŠจๆœบๅ™จไบบ robot = YourRobot() asyncio.run(robot.start()) - + # 2. ่ฎพ็ฝฎๆŠ€่ƒฝ skills = robot.get_skills() skills.add(YourSkill) skills.create_instance("YourSkill", robot=robot) - + # 3. ่ฎพ็ฝฎๅ“ๅบ”ๅผๆต agent_response_subject = rx.subject.Subject() agent_response_stream = agent_response_subject.pipe(ops.share()) - + # 4. ๅˆ›ๅปบ Web ็•Œ้ข web_interface = RobotWebInterface( port=5555, text_streams={"agent_responses": agent_response_stream}, audio_subject=rx.subject.Subject() ) - + # 5. ๅˆ›ๅปบๆ™บ่ƒฝไฝ“ agent = ClaudeAgent( dev_name="your_agent", @@ -167,12 +167,12 @@ def main(): system_query="ๆ‚จ็š„็ณป็ปŸๆ็คบ่ฏ", model_name="claude-3-5-haiku-latest" ) - + # 6. ่ฟžๆŽฅๆ™บ่ƒฝไฝ“ๅ“ๅบ” agent.get_response_observable().subscribe( lambda x: agent_response_subject.on_next(x) ) - + # 7. ่ฟ่กŒ็•Œ้ข web_interface.run() ``` @@ -205,51 +205,51 @@ class MyRobot: self.camera = None self.manipulation = None self.skill_library = SkillLibrary() - + self.capabilities = robot_capabilities or [ RobotCapability.VISION, RobotCapability.MANIPULATION, ] - + async def start(self): # ๅฏๅŠจ DIMOS self.dimos = core.start(2) - + # ๅฏ็”จ LCM pubsub.lcm.autoconf() - + # ้ƒจ็ฝฒ็›ธๆœบ self.camera = self.dimos.deploy( CameraModule, camera_id=0, fps=30 ) - + # ้…็ฝฎ็›ธๆœบ LCM self.camera.color_image.transport = core.LCMTransport("/camera/rgb", Image) self.camera.depth_image.transport = core.LCMTransport("/camera/depth", Image) self.camera.camera_info.transport = core.LCMTransport("/camera/info", CameraInfo) - + # ้ƒจ็ฝฒๆ“ไฝœๆจกๅ— self.manipulation = self.dimos.deploy(ManipulationModule) - + # ่ฟžๆŽฅๆจกๅ— self.manipulation.rgb_image.connect(self.camera.color_image) self.manipulation.depth_image.connect(self.camera.depth_image) self.manipulation.camera_info.connect(self.camera.camera_info) - + # ้…็ฝฎๆ“ไฝœ่พ“ๅ‡บ self.manipulation.viz_image.transport = core.LCMTransport("/viz/output", Image) - + # ๅฏๅŠจๆจกๅ— self.camera.start() self.manipulation.start() - + await asyncio.sleep(2) # ๅ…่ฎธๅˆๅง‹ๅŒ– - + def get_skills(self): return self.skill_library - + def stop(self): if self.manipulation: self.manipulation.stop() @@ -279,29 +279,29 @@ def main(): if not os.getenv("ANTHROPIC_API_KEY"): print("่ฏท่ฎพ็ฝฎ ANTHROPIC_API_KEY") return - + # ๅˆ›ๅปบๆœบๅ™จไบบ robot = MyRobot() - + try: # ๅฏๅŠจๆœบๅ™จไบบ asyncio.run(robot.start()) - + # ่ฎพ็ฝฎๆŠ€่ƒฝ skills = robot.get_skills() skills.add(BasicSkill) skills.create_instance("BasicSkill", robot=robot) - + # ่ฎพ็ฝฎๆต agent_response_subject = rx.subject.Subject() agent_response_stream = agent_response_subject.pipe(ops.share()) - + # ๅˆ›ๅปบ Web ็•Œ้ข web_interface = RobotWebInterface( port=5555, text_streams={"agent_responses": agent_response_stream} ) - + # ๅˆ›ๅปบๆ™บ่ƒฝไฝ“ agent = ClaudeAgent( dev_name="my_agent", @@ -309,17 +309,17 @@ def main(): skills=skills, system_query=SYSTEM_PROMPT ) - + # ่ฟžๆŽฅๅ“ๅบ” agent.get_response_observable().subscribe( lambda x: agent_response_subject.on_next(x) ) - + print("ๆœบๅ™จไบบๅฐฑ็ปช๏ผŒ่ฎฟ้—ฎ http://localhost:5555") - + # ่ฟ่กŒ web_interface.run() - + finally: robot.stop() @@ -341,7 +341,7 @@ from dimos.skills import Skill, skill class BasicSkill(Skill): def __init__(self, robot): self.robot = robot - + def run(self, action: str): # ๅฎž็ŽฐๆŠ€่ƒฝ้€ป่พ‘ return f"ๅทฒๆ‰ง่กŒ๏ผš{action}" @@ -382,27 +382,27 @@ from dimos.core import Module, In, Out, rpc class CustomModule(Module): # ๅฎšไน‰่พ“ๅ…ฅ input_data: In[DataType] = None - - # ๅฎšไน‰่พ“ๅ‡บ + + # ๅฎšไน‰่พ“ๅ‡บ output_data: Out[DataType] = None - + def __init__(self, param1, param2, **kwargs): super().__init__(**kwargs) self.param1 = param1 self.param2 = param2 - + @rpc def start(self): """ๅฏๅŠจๆจกๅ—ๅค„็†ใ€‚""" self.input_data.subscribe(self._process_data) - + def _process_data(self, data): """ๅค„็†่พ“ๅ…ฅๆ•ฐๆฎใ€‚""" # ๅค„็†้€ป่พ‘ result = self.process(data) # ๅ‘ๅธƒ่พ“ๅ‡บ self.output_data.publish(result) - + @rpc def stop(self): """ๅœๆญขๆจกๅ—ใ€‚""" @@ -429,25 +429,25 @@ class ComplexSkill(Skill): def __init__(self, robot, **kwargs): super().__init__(**kwargs) self.robot = robot - + def run(self, target: str, location: Optional[str] = None): """ๆ‰ง่กŒๆŠ€่ƒฝ้€ป่พ‘ใ€‚""" try: # 1. ๆ„Ÿ็Ÿฅ้˜ถๆฎต object_info = self.robot.detect_object(target) - + # 2. ่ง„ๅˆ’้˜ถๆฎต if location: plan = self.robot.plan_movement(object_info, location) - + # 3. ๆ‰ง่กŒ้˜ถๆฎต result = self.robot.execute_plan(plan) - + return { "success": True, "message": f"ๆˆๅŠŸ็งปๅŠจ {target} ๅˆฐ {location}" } - + except Exception as e: return { "success": False, @@ -462,4 +462,4 @@ class ComplexSkill(Skill): 3. **ๅปถ่ฟŸๅŠ ่ฝฝ**๏ผšไป…ๅœจ้œ€่ฆๆ—ถๅˆๅง‹ๅŒ–้‡ๅž‹ๆจกๅ— 4. **่ต„ๆบๆฑ ๅŒ–**๏ผš้‡็”จๆ˜‚่ดต็š„่ต„ๆบ๏ผˆๅฆ‚็ฅž็ป็ฝ‘็ปœๆจกๅž‹๏ผ‰ -ๅธŒๆœ›ๆœฌๆŒ‡ๅ—่ƒฝๅธฎๅŠฉๆ‚จๅฟซ้€ŸไธŠๆ‰‹ DIMOS ๆœบๅ™จไบบๅผ€ๅ‘๏ผ \ No newline at end of file +ๅธŒๆœ›ๆœฌๆŒ‡ๅ—่ƒฝๅธฎๅŠฉๆ‚จๅฟซ้€ŸไธŠๆ‰‹ DIMOS ๆœบๅ™จไบบๅผ€ๅ‘๏ผ diff --git a/dimos/robot/unitree_webrtc/params/front_camera_720.yaml b/dimos/robot/unitree_webrtc/params/front_camera_720.yaml index eb09710667..0030d5fc6c 100644 --- a/dimos/robot/unitree_webrtc/params/front_camera_720.yaml +++ b/dimos/robot/unitree_webrtc/params/front_camera_720.yaml @@ -23,4 +23,4 @@ projection_matrix: cols: 4 data: [651.42609, 0. , 633.16224, 0. , 0. , 804.93951, 373.8537 , 0. , - 0. , 0. , 1. , 0. ] \ No newline at end of file + 0. , 0. , 1. , 0. ] diff --git a/dimos/robot/unitree_webrtc/params/sim_camera.yaml b/dimos/robot/unitree_webrtc/params/sim_camera.yaml index 8fc1574953..6a5ac3e6d8 100644 --- a/dimos/robot/unitree_webrtc/params/sim_camera.yaml +++ b/dimos/robot/unitree_webrtc/params/sim_camera.yaml @@ -23,4 +23,4 @@ projection_matrix: cols: 4 data: [277., 0. , 160. , 0. , 0. , 277., 120. , 0. , - 0. , 0. , 1. , 0. ] \ No newline at end of file + 0. , 0. , 1. , 0. ] diff --git a/dimos/robot/unitree_webrtc/unitree_b1/README.md b/dimos/robot/unitree_webrtc/unitree_b1/README.md index 8616fc286a..f59e6a57ae 100644 --- a/dimos/robot/unitree_webrtc/unitree_b1/README.md +++ b/dimos/robot/unitree_webrtc/unitree_b1/README.md @@ -44,7 +44,7 @@ The B1 robot runs Ubuntu with the following requirements: ```bash # Edit the CMakeLists.txt in the unitree_legged_sdk_B1 directory vim CMakeLists.txt - + # Add this line with the other add_executable statements: add_executable(joystick_server example/joystick_server_udp.cpp) target_link_libraries(joystick_server ${EXTRA_LIBS})``` @@ -144,7 +144,7 @@ This prints commands instead of sending UDP packets - useful for development. - **Mode safety**: Movement only allowed in WALK mode - **Graceful shutdown**: Sends stop commands on exit -### Server Side +### Server Side - **Packet timeout**: Robot stops if no packets for 100ms - **Continuous monitoring**: Checks timeout before every control update - **Safe defaults**: Starts in IDLE mode @@ -173,13 +173,13 @@ External Machine (Client) B1 Robot (Server) This is because the onboard hardware (mini PC, jetson, etc.) needs to connect to both the B1 wifi AP network to send cmd_vel messages over UDP, as well as the network running dimensional -Plug in wireless adapter +Plug in wireless adapter ```bash nmcli device status nmcli device wifi list ifname *DEVICE_NAME* # Connect to b1 network nmcli device wifi connect "Unitree_B1-251" password "00000000" ifname *DEVICE_NAME* -# Verify connection +# Verify connection nmcli connection show --active ``` diff --git a/dimos/robot/unitree_webrtc/unitree_b1/joystick_server_udp.cpp b/dimos/robot/unitree_webrtc/unitree_b1/joystick_server_udp.cpp index 56e2b29412..e86e999b8d 100644 --- a/dimos/robot/unitree_webrtc/unitree_b1/joystick_server_udp.cpp +++ b/dimos/robot/unitree_webrtc/unitree_b1/joystick_server_udp.cpp @@ -31,7 +31,7 @@ struct NetworkJoystickCmd { class JoystickServer { public: - JoystickServer(uint8_t level, int server_port) : + JoystickServer(uint8_t level, int server_port) : safe(LeggedType::B1), udp(level, 8090, "192.168.123.220", 8082), server_port_(server_port), @@ -57,28 +57,28 @@ class JoystickServer { UDP udp; HighCmd cmd = {0}; HighState state = {0}; - + NetworkJoystickCmd joystick_cmd_; std::mutex cmd_mutex_; - + int server_port_; int server_socket_; bool running_; std::thread server_thread_; - + // Client tracking for debug struct sockaddr_in last_client_addr_; bool has_client_ = false; - + // SAFETY: Timeout tracking std::chrono::steady_clock::time_point last_packet_time_; const int PACKET_TIMEOUT_MS = 100; // Stop if no packet for 100ms - + float dt = 0.002; - + // Control parameters const float MAX_FORWARD_SPEED = 0.2f; // m/s - const float MAX_SIDE_SPEED = 0.2f; // m/s + const float MAX_SIDE_SPEED = 0.2f; // m/s const float MAX_YAW_SPEED = 0.2f; // rad/s const float MAX_BODY_HEIGHT = 0.1f; // m const float MAX_EULER_ANGLE = 0.3f; // rad @@ -87,13 +87,13 @@ class JoystickServer { void JoystickServer::Start() { running_ = true; - + // Start network server thread server_thread_ = std::thread(&JoystickServer::NetworkServerThread, this); - + // Initialize environment InitEnvironment(); - + // Start control loops LoopFunc loop_control("control_loop", dt, boost::bind(&JoystickServer::RobotControl, this)); LoopFunc loop_udpSend("udp_send", dt, 3, boost::bind(&JoystickServer::UDPSend, this)); @@ -107,7 +107,7 @@ void JoystickServer::Start() { std::cout << "Timeout protection: " << PACKET_TIMEOUT_MS << "ms" << std::endl; std::cout << "Expected packet size: 19 bytes" << std::endl; std::cout << "Robot control loops started" << std::endl; - + // Keep running while (running_) { sleep(1); @@ -129,41 +129,41 @@ void JoystickServer::NetworkServerThread() { std::cerr << "Failed to create UDP socket" << std::endl; return; } - + // Allow socket reuse int opt = 1; setsockopt(server_socket_, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); - + // Bind socket struct sockaddr_in server_addr; server_addr.sin_family = AF_INET; server_addr.sin_addr.s_addr = INADDR_ANY; server_addr.sin_port = htons(server_port_); - + if (bind(server_socket_, (struct sockaddr*)&server_addr, sizeof(server_addr)) < 0) { std::cerr << "Failed to bind UDP socket to port " << server_port_ << std::endl; close(server_socket_); return; } - + std::cout << "UDP server listening on port " << server_port_ << std::endl; std::cout << "Waiting for joystick packets..." << std::endl; - + NetworkJoystickCmd net_cmd; struct sockaddr_in client_addr; socklen_t client_len; - + while (running_) { client_len = sizeof(client_addr); - + // Receive UDP datagram (blocks until packet arrives) - ssize_t bytes = recvfrom(server_socket_, &net_cmd, sizeof(net_cmd), + ssize_t bytes = recvfrom(server_socket_, &net_cmd, sizeof(net_cmd), 0, (struct sockaddr*)&client_addr, &client_len); - + if (bytes == 19) { // Perfect packet size from Python client if (!has_client_) { - std::cout << "Client connected from " << inet_ntoa(client_addr.sin_addr) + std::cout << "Client connected from " << inet_ntoa(client_addr.sin_addr) << ":" << ntohs(client_addr.sin_port) << std::endl; has_client_ = true; last_client_addr_ = client_addr; @@ -172,7 +172,7 @@ void JoystickServer::NetworkServerThread() { } else if (bytes == sizeof(NetworkJoystickCmd)) { // C++ client with padding (20 bytes) if (!has_client_) { - std::cout << "C++ Client connected from " << inet_ntoa(client_addr.sin_addr) + std::cout << "C++ Client connected from " << inet_ntoa(client_addr.sin_addr) << ":" << ntohs(client_addr.sin_port) << std::endl; has_client_ = true; last_client_addr_ = client_addr; @@ -182,7 +182,7 @@ void JoystickServer::NetworkServerThread() { // Wrong packet size - ignore but log static int error_count = 0; if (error_count++ < 5) { // Only log first 5 errors - std::cerr << "Ignored packet with wrong size: " << bytes + std::cerr << "Ignored packet with wrong size: " << bytes << " bytes (expected 19)" << std::endl; } } @@ -193,10 +193,10 @@ void JoystickServer::NetworkServerThread() { void JoystickServer::ParseJoystickCommand(const NetworkJoystickCmd& net_cmd) { std::lock_guard lock(cmd_mutex_); joystick_cmd_ = net_cmd; - + // SAFETY: Update timestamp for timeout tracking last_packet_time_ = std::chrono::steady_clock::now(); - + // Apply deadzone to analog sticks if (fabs(joystick_cmd_.lx) < DEADZONE) joystick_cmd_.lx = 0; if (fabs(joystick_cmd_.ly) < DEADZONE) joystick_cmd_.ly = 0; @@ -208,16 +208,16 @@ void JoystickServer::CheckTimeout() { auto now = std::chrono::steady_clock::now(); auto elapsed = std::chrono::duration_cast( now - last_packet_time_).count(); - + static bool timeout_printed = false; - + if (elapsed > PACKET_TIMEOUT_MS) { joystick_cmd_.lx = 0; joystick_cmd_.ly = 0; joystick_cmd_.rx = 0; joystick_cmd_.ry = 0; joystick_cmd_.buttons = 0; - + if (!timeout_printed) { std::cout << "SAFETY: Packet timeout - stopping movement!" << std::endl; timeout_printed = true; @@ -241,7 +241,7 @@ void JoystickServer::UDPSend() { void JoystickServer::RobotControl() { udp.GetRecv(state); - + // SAFETY: Check for packet timeout NetworkJoystickCmd current_cmd; { @@ -249,7 +249,7 @@ void JoystickServer::RobotControl() { CheckTimeout(); // This may zero movement if timeout current_cmd = joystick_cmd_; } - + cmd.mode = 0; cmd.gaitType = 0; cmd.speedLevel = 0; @@ -262,35 +262,35 @@ void JoystickServer::RobotControl() { cmd.velocity[1] = 0.0f; cmd.yawSpeed = 0.0f; cmd.reserve = 0; - + // Set mode from joystick cmd.mode = current_cmd.mode; - + // Map joystick to robot control based on mode switch (current_cmd.mode) { case 0: // Idle // Robot stops break; - + case 1: // Force stand with body control // Left stick controls body height and yaw cmd.bodyHeight = current_cmd.ly * MAX_BODY_HEIGHT; cmd.euler[2] = current_cmd.lx * MAX_EULER_ANGLE; - + // Right stick controls pitch and roll cmd.euler[1] = current_cmd.ry * MAX_EULER_ANGLE; cmd.euler[0] = current_cmd.rx * MAX_EULER_ANGLE; break; - + case 2: // Walk mode cmd.velocity[0] = std::clamp(current_cmd.ly * MAX_FORWARD_SPEED, -MAX_FORWARD_SPEED, MAX_FORWARD_SPEED); cmd.yawSpeed = std::clamp(-current_cmd.lx * MAX_YAW_SPEED, -MAX_YAW_SPEED, MAX_YAW_SPEED); cmd.velocity[1] = std::clamp(-current_cmd.rx * MAX_SIDE_SPEED, -MAX_SIDE_SPEED, MAX_SIDE_SPEED); - + // Check button states for gait type if (current_cmd.buttons & 0x0001) { // Button A cmd.gaitType = 0; // Trot - } else if (current_cmd.buttons & 0x0002) { // Button B + } else if (current_cmd.buttons & 0x0002) { // Button B cmd.gaitType = 1; // Trot running } else if (current_cmd.buttons & 0x0004) { // Button X cmd.gaitType = 2; // Climb mode @@ -298,30 +298,30 @@ void JoystickServer::RobotControl() { cmd.gaitType = 3; // Trot obstacle } break; - + case 5: // Damping mode case 6: // Recovery stand up break; - + default: cmd.mode = 0; // Default to idle for safety break; } - + // Debug output static int counter = 0; if (counter++ % 500 == 0) { // Print every second auto now = std::chrono::steady_clock::now(); auto elapsed = std::chrono::duration_cast( now - last_packet_time_).count(); - - std::cout << "Mode: " << (int)cmd.mode + + std::cout << "Mode: " << (int)cmd.mode << " Vel: [" << cmd.velocity[0] << ", " << cmd.velocity[1] << "]" << " Yaw: " << cmd.yawSpeed << " Last packet: " << elapsed << "ms ago" << " IMU: " << state.imu.rpy[2] << std::endl; } - + udp.SetSend(cmd); } @@ -338,11 +338,11 @@ void signal_handler(int sig) { int main(int argc, char* argv[]) { int port = 9090; // Default port - + if (argc > 1) { port = atoi(argv[1]); } - + std::cout << "UDP Unitree B1 Joystick Control Server" << std::endl; std::cout << "Communication level: HIGH-level" << std::endl; std::cout << "Protocol: UDP (datagram)" << std::endl; @@ -352,15 +352,15 @@ int main(int argc, char* argv[]) { std::cout << "WARNING: Make sure the robot is standing on the ground." << std::endl; std::cout << "Press Enter to continue..." << std::endl; std::cin.ignore(); - + JoystickServer server(HIGHLEVEL, port); g_server = &server; - + // Set up signal handler signal(SIGINT, signal_handler); signal(SIGTERM, signal_handler); - + server.Start(); - + return 0; -} \ No newline at end of file +} diff --git a/dimos/simulation/README.md b/dimos/simulation/README.md index 7304e45bf4..95d8b4cda1 100644 --- a/dimos/simulation/README.md +++ b/dimos/simulation/README.md @@ -91,8 +91,8 @@ This will: ## Viewing the Stream -The camera stream will be available at: +The camera stream will be available at: - RTSP: `rtsp://localhost:8554/stream` or `rtsp://:8554/stream` -You can view it using VLC or any RTSP-capable player. \ No newline at end of file +You can view it using VLC or any RTSP-capable player. diff --git a/dimos/skills/unitree/__init__.py b/dimos/skills/unitree/__init__.py index 8b13789179..e69de29bb2 100644 --- a/dimos/skills/unitree/__init__.py +++ b/dimos/skills/unitree/__init__.py @@ -1 +0,0 @@ - diff --git a/dimos/web/README.md b/dimos/web/README.md index 943d7551f9..c7bcd5df20 100644 --- a/dimos/web/README.md +++ b/dimos/web/README.md @@ -93,7 +93,7 @@ from dimos.web.robot_web_interface import RobotWebInterface robot_ip = os.getenv("ROBOT_IP") # Initialize robot -logger.info("Initializing Unitree Robot") +logger.info("Initializing Unitree Robot") robot = UnitreeGo2(ip=robot_ip, connection_method=connection_method, output_dir=output_dir) diff --git a/dimos/web/command-center-extension/src/components/LeafletMap.tsx b/dimos/web/command-center-extension/src/components/LeafletMap.tsx index 79ba4b25da..d0ad2380c4 100644 --- a/dimos/web/command-center-extension/src/components/LeafletMap.tsx +++ b/dimos/web/command-center-extension/src/components/LeafletMap.tsx @@ -147,4 +147,4 @@ const leafletCss = ` } `; -export default LeafletMap; \ No newline at end of file +export default LeafletMap; diff --git a/dimos/web/dimos_interface/api/README.md b/dimos/web/dimos_interface/api/README.md index 38fd275e8a..37cafd6e52 100644 --- a/dimos/web/dimos_interface/api/README.md +++ b/dimos/web/dimos_interface/api/README.md @@ -31,7 +31,7 @@ The server will start on `http://0.0.0.0:5555`. ## Integration with DIMOS Agents -See DimOS Documentation for more info. +See DimOS Documentation for more info. ```python from dimos.agents.agent import OpenAIAgent @@ -42,7 +42,7 @@ from dimos.web.robot_web_interface import RobotWebInterface robot_ip = os.getenv("ROBOT_IP") # Initialize robot -logger.info("Initializing Unitree Robot") +logger.info("Initializing Unitree Robot") robot = UnitreeGo2(ip=robot_ip, connection_method=connection_method, output_dir=output_dir) @@ -83,4 +83,4 @@ The frontend and backend are separate applications: 3. Vite's development server proxies requests from `/unitree/*` to the FastAPI server 4. The `unitree` command in the terminal interface sends requests to these endpoints -This architecture allows the frontend and backend to be developed and run independently. \ No newline at end of file +This architecture allows the frontend and backend to be developed and run independently. diff --git a/dimos/web/dimos_interface/api/requirements.txt b/dimos/web/dimos_interface/api/requirements.txt index a906146c35..a1ab33e428 100644 --- a/dimos/web/dimos_interface/api/requirements.txt +++ b/dimos/web/dimos_interface/api/requirements.txt @@ -4,4 +4,4 @@ reactivex==4.0.4 numpy<2.0.0 # Specify older NumPy version for cv2 compatibility opencv-python==4.8.1.78 python-multipart==0.0.6 -jinja2==3.1.2 \ No newline at end of file +jinja2==3.1.2 diff --git a/dimos/web/dimos_interface/api/templates/index_fastapi.html b/dimos/web/dimos_interface/api/templates/index_fastapi.html index 406557c04a..4cfe943fc7 100644 --- a/dimos/web/dimos_interface/api/templates/index_fastapi.html +++ b/dimos/web/dimos_interface/api/templates/index_fastapi.html @@ -126,13 +126,13 @@ border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); } - + .query-form { display: flex; gap: 10px; align-items: center; } - + .query-input { flex-grow: 1; padding: 10px; @@ -140,7 +140,7 @@ border-radius: 5px; font-size: 16px; } - + .query-button { padding: 10px 20px; background-color: #28a745; @@ -151,11 +151,11 @@ font-size: 16px; transition: background-color 0.3s; } - + .query-button:hover { background-color: #218838; } - + /* Voice button styles */ .voice-button { width: 50px; @@ -173,17 +173,17 @@ box-shadow: 0 2px 5px rgba(0,0,0,0.2); position: relative; } - + .voice-button:hover { transform: scale(1.1); box-shadow: 0 4px 8px rgba(0,0,0,0.3); } - + .voice-button.recording { background-color: #ff0000; animation: pulse 1.5s infinite; } - + .voice-button.recording::after { content: ''; position: absolute; @@ -195,13 +195,13 @@ border-radius: 50%; animation: ripple 1.5s infinite; } - + @keyframes pulse { 0% { transform: scale(1); } 50% { transform: scale(1.05); } 100% { transform: scale(1); } } - + @keyframes ripple { 0% { transform: scale(1); @@ -212,7 +212,7 @@ opacity: 0; } } - + .voice-status { position: absolute; top: -25px; @@ -226,34 +226,34 @@ white-space: nowrap; display: none; } - + .voice-button.recording .voice-status { display: block; } - + .query-response { margin-top: 15px; padding: 10px; border-radius: 5px; display: none; } - + .success { background-color: #d4edda; color: #155724; } - + .error { background-color: #f8d7da; color: #721c24; } - + .text-streams-container { max-width: 800px; margin: 30px auto; } - + .text-stream-container { background: white; padding: 15px; @@ -261,12 +261,12 @@ border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); } - + .text-stream-container h3 { margin-top: 0; color: #444; } - + .text-messages { height: 200px; overflow-y: auto; @@ -276,7 +276,7 @@ margin-bottom: 10px; background-color: #f9f9f9; } - + .text-message { padding: 8px; margin-bottom: 8px; @@ -288,7 +288,7 @@

Live Video Streams

- +

Ask a Question

@@ -303,7 +303,7 @@

Ask a Question

- + {% if text_stream_keys %}
@@ -377,28 +377,28 @@

{{ key.replace('_', ' ').title() }}

if (!mediaRecorder) { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); mediaRecorder = new MediaRecorder(stream); - + mediaRecorder.ondataavailable = e => chunks.push(e.data); - + mediaRecorder.onstop = async () => { const blob = new Blob(chunks, { type: 'audio/webm' }); chunks = []; - + // Show uploading status queryResponse.textContent = 'Processing voice command...'; queryResponse.className = 'query-response'; queryResponse.style.display = 'block'; - + const formData = new FormData(); formData.append('file', blob, 'recording.webm'); - + try { - const res = await fetch('/upload_audio', { - method: 'POST', - body: formData + const res = await fetch('/upload_audio', { + method: 'POST', + body: formData }); const json = await res.json(); - + if (json.success) { queryResponse.textContent = 'Voice command received!'; queryResponse.className = 'query-response success'; @@ -415,7 +415,7 @@

{{ key.replace('_', ' ').title() }}

} }; } - + mediaRecorder.start(); voiceBtn.classList.add('recording'); } catch (err) { @@ -428,24 +428,24 @@

{{ key.replace('_', ' ').title() }}

// Handle query form submission document.getElementById('queryForm').addEventListener('submit', async function(e) { e.preventDefault(); - + const queryInput = document.getElementById('queryInput'); const queryResponse = document.getElementById('queryResponse'); const query = queryInput.value.trim(); - + if (!query) return; - + try { const formData = new FormData(); formData.append('query', query); - + const response = await fetch('/submit_query', { method: 'POST', body: formData }); - + let result; - + // Better error handling for non-200 responses if (!response.ok) { try { @@ -463,14 +463,14 @@

{{ key.replace('_', ' ').title() }}

} else { result = await response.json(); } - + queryResponse.textContent = result.message; queryResponse.className = 'query-response ' + (result.success ? 'success' : 'error'); queryResponse.style.display = 'block'; - + if (result.success) { queryInput.value = ''; - + // Hide the success message after 3 seconds setTimeout(() => { queryResponse.style.display = 'none'; @@ -483,20 +483,20 @@

{{ key.replace('_', ' ').title() }}

} }); - + // Text stream event sources const textEventSources = {}; - + function connectTextStream(key) { // Close if already open if (textEventSources[key]) { textEventSources[key].close(); } - + // Connect to the server-sent events endpoint const eventSource = new EventSource(`/text_stream/${key}`); textEventSources[key] = eventSource; - + // Handle incoming messages eventSource.addEventListener('message', function(event) { const messagesContainer = document.getElementById(`text_messages_${key}`); @@ -504,11 +504,11 @@

{{ key.replace('_', ' ').title() }}

messageElement.className = 'text-message'; messageElement.textContent = event.data; messagesContainer.appendChild(messageElement); - + // Scroll to the bottom to show latest message messagesContainer.scrollTop = messagesContainer.scrollHeight; }); - + // Handle connection errors eventSource.onerror = function() { console.error(`Error in text stream ${key}`); @@ -516,26 +516,26 @@

{{ key.replace('_', ' ').title() }}

delete textEventSources[key]; }; } - + function disconnectTextStream(key) { if (textEventSources[key]) { textEventSources[key].close(); delete textEventSources[key]; } } - + function clearTextStream(key) { const messagesContainer = document.getElementById(`text_messages_${key}`); messagesContainer.innerHTML = ''; } - + // Auto-connect text streams on page load window.addEventListener('load', function() { {% for key in text_stream_keys %} connectTextStream('{{ key }}'); {% endfor %} }); - + - \ No newline at end of file + diff --git a/dimos/web/dimos_interface/src/App.svelte b/dimos/web/dimos_interface/src/App.svelte index c249f3e3ea..8ca51f866d 100644 --- a/dimos/web/dimos_interface/src/App.svelte +++ b/dimos/web/dimos_interface/src/App.svelte @@ -10,17 +10,17 @@ const handleVoiceCommand = async (event: CustomEvent) => { if (event.detail.success) { // Show voice processing message - history.update(h => [...h, { - command: '[voice command]', - outputs: ['Processing voice command...'] + history.update(h => [...h, { + command: '[voice command]', + outputs: ['Processing voice command...'] }]); - + // The actual command will be processed by the agent through the audio pipeline // and will appear in the text stream } else { - history.update(h => [...h, { - command: '[voice command]', - outputs: [`Error: ${event.detail.error}`] + history.update(h => [...h, { + command: '[voice command]', + outputs: [`Error: ${event.detail.error}`] }]); } }; diff --git a/dimos/web/dimos_interface/src/app.css b/dimos/web/dimos_interface/src/app.css index d564a656ea..e85299c54f 100644 --- a/dimos/web/dimos_interface/src/app.css +++ b/dimos/web/dimos_interface/src/app.css @@ -47,4 +47,4 @@ ::-webkit-scrollbar-thumb:hover { background: #555; -} \ No newline at end of file +} diff --git a/dimos/web/dimos_interface/src/components/StreamViewer.svelte b/dimos/web/dimos_interface/src/components/StreamViewer.svelte index 08cf937299..43fe4739dd 100644 --- a/dimos/web/dimos_interface/src/components/StreamViewer.svelte +++ b/dimos/web/dimos_interface/src/components/StreamViewer.svelte @@ -27,15 +27,15 @@ function retryConnection(streamKey: string) { if (!retryCount[streamKey]) retryCount[streamKey] = 0; - + if (retryCount[streamKey] < MAX_RETRIES) { retryCount[streamKey]++; const timeLeft = TOTAL_TIMEOUT - (retryCount[streamKey] * RETRY_INTERVAL); errorMessages[streamKey] = `Connection attempt ${retryCount[streamKey]}/${MAX_RETRIES}... (${Math.ceil(timeLeft / 1000)}s remaining)`; - + // Update timestamp to force a new connection attempt timestamps[streamKey] = Date.now(); - + clearRetryTimer(streamKey); retryTimers[streamKey] = setTimeout(() => retryConnection(streamKey), RETRY_INTERVAL); } else { @@ -193,4 +193,4 @@ align-items: center; justify-content: center; } - \ No newline at end of file + diff --git a/dimos/web/dimos_interface/src/components/VoiceButton.svelte b/dimos/web/dimos_interface/src/components/VoiceButton.svelte index 0f9682519a..a316836d2e 100644 --- a/dimos/web/dimos_interface/src/components/VoiceButton.svelte +++ b/dimos/web/dimos_interface/src/components/VoiceButton.svelte @@ -44,53 +44,53 @@ if (!mediaRecorder) { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); mediaRecorder = new MediaRecorder(stream); - + mediaRecorder.ondataavailable = (e) => chunks.push(e.data); - + mediaRecorder.onstop = async () => { isProcessing = true; const blob = new Blob(chunks, { type: 'audio/webm' }); chunks = []; - + // Upload to backend const formData = new FormData(); formData.append('file', blob, 'recording.webm'); - + try { const res = await fetch(`${getServerUrl()}/upload_audio`, { method: 'POST', body: formData }); - + const json = await res.json(); - + if (json.success) { // Connect to agent_responses stream to see the output connectTextStream('agent_responses'); dispatch('voiceCommand', { success: true }); } else { - dispatch('voiceCommand', { - success: false, - error: json.message + dispatch('voiceCommand', { + success: false, + error: json.message }); } } catch (err) { - dispatch('voiceCommand', { - success: false, - error: err instanceof Error ? err.message : 'Upload failed' + dispatch('voiceCommand', { + success: false, + error: err instanceof Error ? err.message : 'Upload failed' }); } finally { isProcessing = false; } }; } - + mediaRecorder.start(); isRecording = true; } catch (err) { - dispatch('voiceCommand', { - success: false, - error: 'Microphone access denied' + dispatch('voiceCommand', { + success: false, + error: 'Microphone access denied' }); } } @@ -197,15 +197,15 @@ } @keyframes pulse { - 0% { + 0% { transform: scale(1); box-shadow: 0 4px 12px rgba(255, 0, 0, 0.4); } - 50% { + 50% { transform: scale(1.05); box-shadow: 0 4px 20px rgba(255, 0, 0, 0.6); } - 100% { + 100% { transform: scale(1); box-shadow: 0 4px 12px rgba(255, 0, 0, 0.4); } @@ -259,4 +259,4 @@ font-size: 44px; /* Increased from 22px to 2x */ } } - \ No newline at end of file + diff --git a/dimos/web/dimos_interface/src/stores/stream.ts b/dimos/web/dimos_interface/src/stores/stream.ts index eee46f84bf..649fd515ce 100644 --- a/dimos/web/dimos_interface/src/stores/stream.ts +++ b/dimos/web/dimos_interface/src/stores/stream.ts @@ -95,7 +95,7 @@ fetchAvailableStreams().then(streams => { export const showStream = async (streamKey?: string) => { streamStore.update(state => ({ ...state, isLoading: true, error: null })); - + try { const streams = await fetchAvailableStreams(); if (streams.length === 0) { @@ -178,4 +178,3 @@ export const disconnectTextStream = (key: string): void => { delete textEventSources[key]; } }; - diff --git a/dimos/web/dimos_interface/src/utils/commands.ts b/dimos/web/dimos_interface/src/utils/commands.ts index 455a0092e0..53755630ac 100644 --- a/dimos/web/dimos_interface/src/utils/commands.ts +++ b/dimos/web/dimos_interface/src/utils/commands.ts @@ -182,14 +182,14 @@ export const commands: Record Promise }, banner: () => ` -โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— -โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ -โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ -โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ•šโ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ +โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— +โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ +โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ +โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ•šโ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ•šโ•โ• โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ•โ•šโ•โ• โ•šโ•โ•โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•โ•โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ• โ•šโ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•šโ•โ•โ•โ•โ•โ•โ•v${packageJson.version} -Powering generalist robotics +Powering generalist robotics Type 'help' to see list of available commands. `, @@ -272,7 +272,7 @@ Type 'help' to see list of available commands. } const jointPositions = args.join(' '); - + try { const jointPositionsArray = jointPositions.split(',').map(x => parseFloat(x.trim())); const response = await fetch(`${state.connection.url}/control?t=${Date.now()}`, { @@ -285,7 +285,7 @@ Type 'help' to see list of available commands. }); const data = await response.json(); - + if (response.ok) { return `${data.message} โœ“`; } else { @@ -302,7 +302,7 @@ Type 'help' to see list of available commands. } const subcommand = args[0].toLowerCase(); - + if (subcommand === 'status') { try { const response = await fetch('/unitree/status'); @@ -331,14 +331,14 @@ Type 'help' to see list of available commands. hideStream(); return 'Stopped Unitree video stream.'; } - + if (subcommand === 'command') { if (args.length < 2) { return 'Usage: unitree command - Send a command to the Unitree API'; } - + const commandText = args.slice(1).join(' '); - + try { // Ensure we have the text stream keys if (textStreamKeys.length === 0) { @@ -352,7 +352,7 @@ Type 'help' to see list of available commands. }, body: JSON.stringify({ command: commandText }) }); - + if (!response.ok) { throw new Error(`Server returned ${response.status}`); } @@ -362,13 +362,13 @@ Type 'help' to see list of available commands. streamKey: textStreamKeys[0], // Using the first available text stream initialMessage: `Command sent: ${commandText}\nPlanningAgent output...` }; - + } catch (error) { const message = error instanceof Error ? error.message : 'Server unreachable'; return `Failed to send command: ${message}. Make sure the API server is running.`; } } - + return 'Invalid subcommand. Available subcommands: status, start_stream, stop_stream, command'; }, }; diff --git a/dimos/web/dimos_interface/src/utils/simulation.ts b/dimos/web/dimos_interface/src/utils/simulation.ts index 5373bdb8b8..6e71dda358 100644 --- a/dimos/web/dimos_interface/src/utils/simulation.ts +++ b/dimos/web/dimos_interface/src/utils/simulation.ts @@ -115,7 +115,7 @@ export class SimulationManager { async requestSimulation(): Promise { simulationStore.update(state => ({ ...state, isConnecting: true, error: null })); - + try { // Request instance allocation const response = await this.fetchWithRetry(this.apiEndpoint, { @@ -129,7 +129,7 @@ export class SimulationManager { }); const instanceInfo = await response.json(); - + if (import.meta.env.DEV) { console.log('API Response:', instanceInfo); } @@ -175,11 +175,11 @@ export class SimulationManager { isConnecting: false, error: errorMessage })); - + if (import.meta.env.DEV) { console.error('Simulation request failed:', error); } - + throw error; } } @@ -211,4 +211,4 @@ export class SimulationManager { } } -export const simulationManager = new SimulationManager(); \ No newline at end of file +export const simulationManager = new SimulationManager(); diff --git a/dimos/web/templates/index_fastapi.html b/dimos/web/templates/index_fastapi.html index 9ab54dc170..75b0c1c179 100644 --- a/dimos/web/templates/index_fastapi.html +++ b/dimos/web/templates/index_fastapi.html @@ -125,12 +125,12 @@ border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); } - + .query-form { display: flex; gap: 10px; } - + .query-input { flex-grow: 1; padding: 10px; @@ -138,7 +138,7 @@ border-radius: 5px; font-size: 16px; } - + .query-button { padding: 10px 20px; background-color: #28a745; @@ -149,33 +149,33 @@ font-size: 16px; transition: background-color 0.3s; } - + .query-button:hover { background-color: #218838; } - + .query-response { margin-top: 15px; padding: 10px; border-radius: 5px; display: none; } - + .success { background-color: #d4edda; color: #155724; } - + .error { background-color: #f8d7da; color: #721c24; } - + .text-streams-container { max-width: 800px; margin: 30px auto; } - + .text-stream-container { background: white; padding: 15px; @@ -183,12 +183,12 @@ border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); } - + .text-stream-container h3 { margin-top: 0; color: #444; } - + .text-messages { height: 200px; overflow-y: auto; @@ -198,7 +198,7 @@ margin-bottom: 10px; background-color: #f9f9f9; } - + .text-message { padding: 8px; margin-bottom: 8px; @@ -210,7 +210,7 @@

Live Video Streams

- +

Ask a Question

@@ -219,7 +219,7 @@

Ask a Question

- + {% if text_stream_keys %}
@@ -237,7 +237,7 @@

{{ key.replace('_', ' ').title() }}

{% endfor %}
{% endif %} - +
{% for key in stream_keys %}
@@ -278,24 +278,24 @@

{{ key.replace('_', ' ').title() }}

// Handle query form submission document.getElementById('queryForm').addEventListener('submit', async function(e) { e.preventDefault(); - + const queryInput = document.getElementById('queryInput'); const queryResponse = document.getElementById('queryResponse'); const query = queryInput.value.trim(); - + if (!query) return; - + try { const formData = new FormData(); formData.append('query', query); - + const response = await fetch('/submit_query', { method: 'POST', body: formData }); - + let result; - + // Better error handling for non-200 responses if (!response.ok) { try { @@ -313,14 +313,14 @@

{{ key.replace('_', ' ').title() }}

} else { result = await response.json(); } - + queryResponse.textContent = result.message; queryResponse.className = 'query-response ' + (result.success ? 'success' : 'error'); queryResponse.style.display = 'block'; - + if (result.success) { queryInput.value = ''; - + // Hide the success message after 3 seconds setTimeout(() => { queryResponse.style.display = 'none'; @@ -335,17 +335,17 @@

{{ key.replace('_', ' ').title() }}

// Text stream event sources const textEventSources = {}; - + function connectTextStream(key) { // Close if already open if (textEventSources[key]) { textEventSources[key].close(); } - + // Connect to the server-sent events endpoint const eventSource = new EventSource(`/text_stream/${key}`); textEventSources[key] = eventSource; - + // Handle incoming messages eventSource.addEventListener('message', function(event) { const messagesContainer = document.getElementById(`text_messages_${key}`); @@ -353,11 +353,11 @@

{{ key.replace('_', ' ').title() }}

messageElement.className = 'text-message'; messageElement.textContent = event.data; messagesContainer.appendChild(messageElement); - + // Scroll to the bottom to show latest message messagesContainer.scrollTop = messagesContainer.scrollHeight; }); - + // Handle connection errors eventSource.onerror = function() { console.error(`Error in text stream ${key}`); @@ -365,19 +365,19 @@

{{ key.replace('_', ' ').title() }}

delete textEventSources[key]; }; } - + function disconnectTextStream(key) { if (textEventSources[key]) { textEventSources[key].close(); delete textEventSources[key]; } } - + function clearTextStream(key) { const messagesContainer = document.getElementById(`text_messages_${key}`); messagesContainer.innerHTML = ''; } - + // Auto-connect text streams on page load window.addEventListener('load', function() { {% for key in text_stream_keys %} @@ -386,4 +386,4 @@

{{ key.replace('_', ' ').title() }}

}); - \ No newline at end of file + diff --git a/dimos/web/templates/index_flask.html b/dimos/web/templates/index_flask.html index 4717553d95..e41665e588 100644 --- a/dimos/web/templates/index_flask.html +++ b/dimos/web/templates/index_flask.html @@ -98,7 +98,7 @@

Live Video Streams

- +
{% for key in stream_keys %}
@@ -115,4 +115,4 @@

{{ key.replace('_', ' ').title() }}

} - \ No newline at end of file + diff --git a/docker/dev/bash.sh b/docker/dev/bash.sh index 878faa23c5..c5248841d9 100755 --- a/docker/dev/bash.sh +++ b/docker/dev/bash.sh @@ -60,7 +60,7 @@ function tmpUmask { oldUmask=$(umask) newUmask=$1 - + shift umask $newUmask echo umask $(umask -S) @@ -68,7 +68,7 @@ function tmpUmask eval $@ umask $oldUmask echo umask $(umask -S) - + } function newloginuser diff --git a/docker/dev/tmux.conf b/docker/dev/tmux.conf index aad055fe5a..ecf6b22ced 100644 --- a/docker/dev/tmux.conf +++ b/docker/dev/tmux.conf @@ -35,7 +35,7 @@ bind c new-window -c "#{pane_current_path}" #bind -n C-M-right swap-window -t +1 #set -g default-terminal "screen-256color" #set -g default-terminal "xterm" - + bind-key u capture-pane \; save-buffer /tmp/tmux-buffer \; run-shell "urxvtc --geometry 51x20 --title 'floatme' -e bash -c \"cat /tmp/tmux-buffer | urlview\" " bind-key r source-file ~/.tmux.conf @@ -79,6 +79,6 @@ setw -g window-status-separator "#[bg=colour235]" setw -g window-status-style "fg=colour253,bg=black,none" set -g status-left "" set -g status-right "#[bg=black]#[fg=colour244]#h#[fg=colour244]#[fg=colour3]/#[fg=colour244]#S" - + setw -g window-status-format " #[fg=colour3]#I#[fg=colour244] #W " setw -g window-status-current-format " #[fg=color3]#I#[fg=colour254] #W " diff --git a/docker/navigation/.env.hardware b/docker/navigation/.env.hardware index 75192623e4..05e08bd375 100644 --- a/docker/navigation/.env.hardware +++ b/docker/navigation/.env.hardware @@ -16,7 +16,7 @@ ROS_DOMAIN_ID=42 # Robot configuration ('mechanum_drive', 'unitree/unitree_g1', 'unitree/unitree_g1', etc) ROBOT_CONFIG_PATH=mechanum_drive -# Robot IP address on local network for connection over WebRTC +# Robot IP address on local network for connection over WebRTC # For Unitree Go2, Unitree G1, if using WebRTCConnection # This can be found in the unitree app under Device settings or via network scan ROBOT_IP= diff --git a/docker/navigation/.gitignore b/docker/navigation/.gitignore index 434f1e37bc..0eaccbc740 100644 --- a/docker/navigation/.gitignore +++ b/docker/navigation/.gitignore @@ -17,4 +17,4 @@ config/ *.tmp *.log *.swp -*~ \ No newline at end of file +*~ diff --git a/docker/navigation/README.md b/docker/navigation/README.md index c1d1eba40f..87c5b6e788 100644 --- a/docker/navigation/README.md +++ b/docker/navigation/README.md @@ -115,7 +115,7 @@ LIDAR_INTERFACE=eth0 # Your ethernet interface (find with: ip link LIDAR_COMPUTER_IP=192.168.1.5 # Computer IP on the lidar subnet LIDAR_GATEWAY=192.168.1.1 # Gateway IP address for the lidar subnet LIDAR_IP=192.168.1.116 # Full IP address of your Mid-360 lidar -ROBOT_IP= # IP addres of robot on local network (if using WebRTC connection) +ROBOT_IP= # IP addres of robot on local network (if using WebRTC connection) # Motor Controller MOTOR_SERIAL_DEVICE=/dev/ttyACM0 # Serial device (check with: ls /dev/ttyACM*) @@ -141,18 +141,17 @@ In the container to run the full navigation stack you must run both the dimensio #### Dimensional Python + Connection Module -For the Unitree G1 +For the Unitree G1 ```bash dimos run unitree-g1 -ROBOT_IP=XX.X.X.XXX dimos run unitree-g1 # If ROBOT_IP env variable is not set in .env +ROBOT_IP=XX.X.X.XXX dimos run unitree-g1 # If ROBOT_IP env variable is not set in .env ``` -#### Navigation Stack +#### Navigation Stack ```bash cd /ros2_ws/src/ros-navigation-autonomy-stack ./system_real_robot_with_route_planner.sh ``` -Now you can place goal points/poses in RVIZ by clicking the "Goalpoint" button. The robot will navigate to the point, running both local and global planners for dynamic obstacle avoidance. - +Now you can place goal points/poses in RVIZ by clicking the "Goalpoint" button. The robot will navigate to the point, running both local and global planners for dynamic obstacle avoidance. diff --git a/docker/navigation/run_both.sh b/docker/navigation/run_both.sh index a433cf5b01..24c480eaea 100755 --- a/docker/navigation/run_both.sh +++ b/docker/navigation/run_both.sh @@ -144,4 +144,4 @@ if [ -n "$DIMOS_PID" ]; then wait $ROS_PID $DIMOS_PID 2>/dev/null || true else wait $ROS_PID 2>/dev/null || true -fi \ No newline at end of file +fi diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile index 8acd7a52af..6fbd5545e5 100644 --- a/docker/python/Dockerfile +++ b/docker/python/Dockerfile @@ -49,4 +49,4 @@ COPY . /app/ # Install dependencies with UV (10-100x faster than pip) RUN uv pip install --upgrade 'pip>=24' 'setuptools>=70' 'wheel' 'packaging>=24' && \ - uv pip install '.[cpu]' \ No newline at end of file + uv pip install '.[cpu]' diff --git a/docker/ros/Dockerfile b/docker/ros/Dockerfile index 22bb3ed547..2dc2b5dbb7 100644 --- a/docker/ros/Dockerfile +++ b/docker/ros/Dockerfile @@ -83,7 +83,7 @@ RUN apt-get update && apt-get install -y \ # Initialize rosdep RUN rosdep init -RUN rosdep update +RUN rosdep update # Source ROS2 and workspace in bashrc RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" >> /root/.bashrc diff --git a/docs/ci.md b/docs/ci.md index a041ab08cc..ac9b11115a 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -50,7 +50,7 @@ pytest # run tests ### Current hierarchy - + ``` โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ubuntuโ”‚ @@ -64,7 +64,7 @@ pytest # run tests โ”Œโ–ฝโ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ros-devโ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` +``` * ghcr.io/dimensionalos/ros:dev * ghcr.io/dimensionalos/python:dev @@ -74,7 +74,7 @@ pytest # run tests > **Note**: The diagram shows only currently active images; the system is extensibleโ€”new combinations are possible, builds can be run per branch and as parallel as possible - + ``` โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ubuntuโ”‚ @@ -135,7 +135,7 @@ Ideally a child job (e.g. **ros-python**) should depend on both: GitHub Actions canโ€™t express โ€œrun only if *both* conditions are true *and* the parent job wasnโ€™t skippedโ€. We are using `needs: [check-changes, ros]` to ensure the job runs after the ros build, but if ros build has been skipped we need `if: always()` to ensure that the build runs anyway. -Adding `always` for some reason completely breaks the conditional check, we cannot have OR, AND operators, it just makes the job _always_ run, which means we build python even if we don't need to. +Adding `always` for some reason completely breaks the conditional check, we cannot have OR, AND operators, it just makes the job _always_ run, which means we build python even if we don't need to. This is unfortunate as the build takes ~30 min first time (a few minutes afterwards thanks to caching) and I've spent a lot of time on this, lots of viable seeming options didn't pan out and probably we need to completely rewrite and own the actions runner and not depend on github structure at all. Single job called `CI` or something, within our custom docker image. diff --git a/docs/development.md b/docs/development.md index 8718144642..838bae6fdb 100644 --- a/docs/development.md +++ b/docs/development.md @@ -22,7 +22,7 @@ Install the *Dev Containers* plug-in for VS Code, Cursor, or your IDE of choice ### Shell only quick start -Terminal within your IDE should use devcontainer transparently given you installed the plugin, but in case you want to run our shell without an IDE, you can use `./bin/dev` +Terminal within your IDE should use devcontainer transparently given you installed the plugin, but in case you want to run our shell without an IDE, you can use `./bin/dev` (it depends on npm/node being installed) ```sh @@ -37,16 +37,16 @@ found 0 vulnerabilities [5299 ms] f0355b6574d9bd277d6eb613e1dc32e3bc18e7493e5b170e335d0e403578bcdb {"outcome":"success","containerId":"f0355b6574d9bd277d6eb613e1dc32e3bc18e7493e5b170e335d0e403578bcdb","remoteUser":"root","remoteWorkspaceFolder":"/workspaces/dimos"} - โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— - โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ - โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ - โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ•šโ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ + โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— + โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ + โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ + โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ•šโ•โ•โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ•šโ•โ• โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ•โ•šโ•โ• โ•šโ•โ•โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•โ•โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ• โ•šโ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•šโ•โ•โ•โ•โ•โ•โ• v_unknown:unknown | Wed May 28 09:23:33 PM UTC 2025 -root@dimos:/workspaces/dimos # +root@dimos:/workspaces/dimos # ``` The script will: @@ -58,7 +58,7 @@ Youโ€™ll land in the workspace as **root** with all project tooling available. ## Pre-Commit Hooks -We use [pre-commit](https://pre-commit.com) (config in `.pre-commit-config.yaml`) to enforce formatting, licence headers, EOLs, LFS checks, etc. Hooks run in **milliseconds**. +We use [pre-commit](https://pre-commit.com) (config in `.pre-commit-config.yaml`) to enforce formatting, licence headers, EOLs, LFS checks, etc. Hooks run in **milliseconds**. Hooks also run in CI; any auto-fixes are committed back to your PR, so local installation is optional โ€” but gives faster feedback. ```sh @@ -74,11 +74,11 @@ format json..............................................................Passed LFS data.................................................................Passed ``` -Given your editor uses ruff via devcontainers (which it should) actual auto-commit hook won't ever reformat your code - IDE will have already done this. +Given your editor uses ruff via devcontainers (which it should) actual auto-commit hook won't ever reformat your code - IDE will have already done this. ### Running hooks manually -Given your editor uses git via devcontainers (which it should) auto-commit hooks will run automatically, this is in case you want to run them manually. +Given your editor uses git via devcontainers (which it should) auto-commit hooks will run automatically, this is in case you want to run them manually. Inside the dev container (Your IDE will likely run this transparently for each commit if using devcontainer plugin): @@ -140,7 +140,7 @@ Classic development run within a subtree: root@dimos:/workspaces/dimos # cd dimos/robot/unitree_webrtc/ root@dimos:/workspaces/dimos/dimos/robot/unitree_webrtc # pytest -collected 27 items / 22 deselected / 5 selected +collected 27 items / 22 deselected / 5 selected type/test_map.py::test_robot_mapping PASSED type/test_timeseries.py::test_repr PASSED @@ -155,13 +155,13 @@ Showing prints: ```sh root@dimos:/workspaces/dimos/dimos/robot/unitree_webrtc/type # pytest -s test_odometry.py test_odometry.py::test_odometry_conversion_and_count Odom ts(2025-05-30 13:52:03) pos(โ†’ Vector Vector([0.432199 0.108042 0.316589])), rot(โ†‘ Vector Vector([ 7.7200000e-04 -9.1280000e-03 3.006 -8621e+00])) yaw(172.3ยฐ) -Odom ts(2025-05-30 13:52:03) pos(โ†’ Vector Vector([0.433629 0.105965 0.316143])), rot(โ†‘ Vector Vector([ 0.003814 -0.006436 2.99591235])) yaw(171.7ยฐ) -Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.434459 0.104739 0.314794])), rot(โ†— Vector Vector([ 0.005558 -0.004183 3.00068456])) yaw(171.9ยฐ) -Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.435621 0.101699 0.315852])), rot(โ†‘ Vector Vector([ 0.005391 -0.006002 3.00246893])) yaw(172.0ยฐ) -Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.436457 0.09857 0.315254])), rot(โ†‘ Vector Vector([ 0.003358 -0.006916 3.00347172])) yaw(172.1ยฐ) +8621e+00])) yaw(172.3ยฐ) +Odom ts(2025-05-30 13:52:03) pos(โ†’ Vector Vector([0.433629 0.105965 0.316143])), rot(โ†‘ Vector Vector([ 0.003814 -0.006436 2.99591235])) yaw(171.7ยฐ) +Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.434459 0.104739 0.314794])), rot(โ†— Vector Vector([ 0.005558 -0.004183 3.00068456])) yaw(171.9ยฐ) +Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.435621 0.101699 0.315852])), rot(โ†‘ Vector Vector([ 0.005391 -0.006002 3.00246893])) yaw(172.0ยฐ) +Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.436457 0.09857 0.315254])), rot(โ†‘ Vector Vector([ 0.003358 -0.006916 3.00347172])) yaw(172.1ยฐ) Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.435535 0.097022 0.314399])), rot(โ†‘ Vector Vector([ 1.88300000e-03 -8.17800000e-03 3.00573432e+00])) yaw(172.2ยฐ) -Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.433739 0.097553 0.313479])), rot(โ†‘ Vector Vector([ 8.10000000e-05 -8.71700000e-03 3.00729616e+00])) yaw(172.3ยฐ) +Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.433739 0.097553 0.313479])), rot(โ†‘ Vector Vector([ 8.10000000e-05 -8.71700000e-03 3.00729616e+00])) yaw(172.3ยฐ) Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.430924 0.09859 0.31322 ])), rot(โ†‘ Vector Vector([ 1.84000000e-04 -9.68700000e-03 3.00945623e+00])) yaw(172.4ยฐ) ... etc ``` @@ -178,5 +178,3 @@ Odom ts(2025-05-30 13:52:04) pos(โ†’ Vector Vector([0.430924 0.09859 0.31322 ]) | Filter tests by name | `pytest -k ""` | | Enable stdout in tests | `pytest -s` | | Run tagged tests | `pytest -m ` | - - diff --git a/docs/jetson.MD b/docs/jetson.MD index 31da4225d9..a4d06e3255 100644 --- a/docs/jetson.MD +++ b/docs/jetson.MD @@ -1,8 +1,8 @@ -# DimOS Jetson Setup Instructions -Tested on Jetpack 6.2, CUDA 12.6 +# DimOS Jetson Setup Instructions +Tested on Jetpack 6.2, CUDA 12.6 -## Required system dependencies -`sudo apt install portaudio19-dev python3-pyaudio` +## Required system dependencies +`sudo apt install portaudio19-dev python3-pyaudio` ## Installing cuSPARSELt https://ninjalabo.ai/blogs/jetson_pytorch.html @@ -17,7 +17,7 @@ ldconfig ``` ## Install Torch and Torchvision wheels -Enter virtualenv +Enter virtualenv ```bash python3 -m venv venv source venv/bin/activate @@ -26,19 +26,19 @@ source venv/bin/activate Wheels for jp6/cu126 https://pypi.jetson-ai-lab.io/jp6/cu126 -Check compatibility: +Check compatibility: https://docs.nvidia.com/deeplearning/frameworks/install-pytorch-jetson-platform-release-notes/pytorch-jetson-rel.html ### Working torch wheel tested on Jetpack 6.2, CUDA 12.6 `pip install --no-cache https://developer.download.nvidia.com/compute/redist/jp/v61/pytorch/torch-2.5.0a0+872d972e41.nv24.08.17622132-cp310-cp310-linux_aarch64.whl` -### Install torchvision from source: +### Install torchvision from source: ```bash -# Set version by checking above torchvision<-->torch compatibility +# Set version by checking above torchvision<-->torch compatibility # We use 0.20.0 export VERSION=20 - + sudo apt-get install libjpeg-dev zlib1g-dev libpython3-dev libopenblas-dev libavcodec-dev libavformat-dev libswscale-dev git clone --branch release/0.$VERSION https://github.com/pytorch/vision torchvision cd torchvision @@ -46,7 +46,7 @@ export BUILD_VERSION=0.$VERSION.0 python3 setup.py install --user # remove --user if installing in virtualenv ``` -### Verify success: +### Verify success: ```bash $ python3 import torch @@ -65,8 +65,8 @@ import torchvision print(torchvision.__version__) ``` -## Install Onnxruntime-gpu +## Install Onnxruntime-gpu Find pre-build wheels here for your specific JP/CUDA version: https://pypi.jetson-ai-lab.io/jp6 -`pip install https://pypi.jetson-ai-lab.io/jp6/cu126/+f/4eb/e6a8902dc7708/onnxruntime_gpu-1.23.0-cp310-cp310-linux_aarch64.whl#sha256=4ebe6a8902dc7708434b2e1541b3fe629ebf434e16ab5537d1d6a622b42c622b` +`pip install https://pypi.jetson-ai-lab.io/jp6/cu126/+f/4eb/e6a8902dc7708/onnxruntime_gpu-1.23.0-cp310-cp310-linux_aarch64.whl#sha256=4ebe6a8902dc7708434b2e1541b3fe629ebf434e16ab5537d1d6a622b42c622b` diff --git a/docs/modules.md b/docs/modules.md index 8ce6d0f5f8..9cdbf586ac 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -13,7 +13,7 @@ from dimos.msgs.geometry_msgs import Vector3 class MyModule(Module): # Declare inputs/outputs as class attributes initialized to None - data_in: In[Vector3] = None + data_in: In[Vector3] = None data_out: Out[Vector3] = None def __init__(): @@ -60,7 +60,7 @@ stream.subscribe(lambda msg: print(f"Received: {msg}")) # Start modules to begin processing module.start() # Calls the @rpc start() method if defined -# Inspect module I/O configuration +# Inspect module I/O configuration print(module.io().result()) # Shows inputs, outputs, and RPC methods # Clean shutdown @@ -107,7 +107,7 @@ pubsub.lcm.autoconf() This architecture enables building complex robotic systems as composable, distributed modules that communicate efficiently via streams and RPC, scaling from single machines to clusters. -# Dimensional Install +# Dimensional Install ## Python Installation (Ubuntu 22.04) ```bash @@ -140,8 +140,8 @@ pip install .[cuda,dev] cp default.env .env ``` -### Test install -```bash +### Test install +```bash # Run standard tests pytest -s dimos/ @@ -163,5 +163,3 @@ export ROBOT_IP= # Run the multiprocess Unitree Go2 example python dimos/robot/unitree_webrtc/multiprocess/unitree_go2.py ``` - - diff --git a/docs/modules_CN.md b/docs/modules_CN.md index d8f088ef59..89e16c7112 100644 --- a/docs/modules_CN.md +++ b/docs/modules_CN.md @@ -60,7 +60,7 @@ stream.subscribe(lambda msg: print(f"ๆŽฅๆ”ถๅˆฐ: {msg}")) # ๅฏๅŠจๆจกๅ—ไปฅๅผ€ๅง‹ๅค„็† module.start() # ๅฆ‚ๆžœๅฎšไน‰ไบ† @rpc start() ๆ–นๆณ•๏ผŒๅˆ™่ฐƒ็”จๅฎƒ -# ๆฃ€ๆŸฅๆจกๅ— I/O ้…็ฝฎ +# ๆฃ€ๆŸฅๆจกๅ— I/O ้…็ฝฎ print(module.io().result()) # ๆ˜พ็คบ่พ“ๅ…ฅใ€่พ“ๅ‡บๅ’Œ RPC ๆ–นๆณ• # ไผ˜้›…ๅ…ณ้—ญ @@ -141,7 +141,7 @@ cp default.env .env ``` ### ๆต‹่ฏ•ๅฎ‰่ฃ… -```bash +```bash # ่ฟ่กŒๆ ‡ๅ‡†ๆต‹่ฏ• pytest -s dimos/ @@ -185,4 +185,4 @@ LCM ไผ ่พ“้’ˆๅฏนๆœบๅ™จไบบๅบ”็”จ่ฟ›่กŒไบ†ไผ˜ๅŒ–๏ผš - **้›ถๆ‹ท่ด**๏ผšๅคงๅž‹ๆถˆๆฏ็š„้ซ˜ๆ•ˆๅ†…ๅญ˜ไฝฟ็”จ - **ไฝŽๅปถ่ฟŸ**๏ผšๅพฎ็ง’็บง็š„ๆถˆๆฏไผ ้€’ -- **ๅคšๆ’ญๆ”ฏๆŒ**๏ผšไธ€ๅฏนๅคš็š„้ซ˜ๆ•ˆ้€šไฟก \ No newline at end of file +- **ๅคšๆ’ญๆ”ฏๆŒ**๏ผšไธ€ๅฏนๅคš็š„้ซ˜ๆ•ˆ้€šไฟก diff --git a/docs/testing_stream_reply.md b/docs/testing_stream_reply.md index f6b76d3ed9..e3189bb5e8 100644 --- a/docs/testing_stream_reply.md +++ b/docs/testing_stream_reply.md @@ -24,7 +24,7 @@ A lightweight framework for **recording, storing, and replaying binary data stre * **No repo bloat** โ€“ binaries live in Gitย LFS; the working tree stays trim. * **Symmetric API** โ€“ `SensorReplay` โ†”๏ธŽ `SensorStorage`; same name, different direction. * **Format agnostic** โ€“ replay *anything* you can pickle (protobuf, numpy, JPEG, โ€ฆ). -* **Data type agnostic** โ€“ with testData("raw_odometry_rotate_walk") you get a Path object back, can be a raw video file, whole codebase, ML model etc +* **Data type agnostic** โ€“ with testData("raw_odometry_rotate_walk") you get a Path object back, can be a raw video file, whole codebase, ML model etc --- @@ -172,4 +172,3 @@ Either delete or run ./bin/lfs_push * `dimos/robot/unitree_webrtc/type/test_odometry.py` * `dimos/robot/unitree_webrtc/type/test_map.py` - diff --git a/flake.nix b/flake.nix index 0061153089..0e8dbe60a1 100644 --- a/flake.nix +++ b/flake.nix @@ -36,7 +36,7 @@ ### GTK / OpenCV helpers glib gtk3 gdk-pixbuf gobject-introspection - + ### GStreamer gst_all_1.gstreamer gst_all_1.gst-plugins-base gst_all_1.gst-plugins-good gst_all_1.gst-plugins-bad gst_all_1.gst-plugins-ugly @@ -44,7 +44,7 @@ ### Open3D & build-time eigen cmake ninja jsoncpp libjpeg libpng - + ### LCM (Lightweight Communications and Marshalling) lcm ]; diff --git a/pyproject.toml b/pyproject.toml index 4f2d866ffa..dae854c0f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,25 +46,25 @@ dependencies = [ "asyncio==3.4.3", "go2-webrtc-connect @ git+https://github.com/dimensionalOS/go2_webrtc_connect.git", "tensorzero==2025.7.5", - + # Web Extensions "fastapi>=0.115.6", "sse-starlette>=2.2.1", "uvicorn>=0.34.0", - + # Agents "langchain>=0.3.27", "langchain-chroma>=0.2.5", "langchain-core>=0.3.72", "langchain-openai>=0.3.28", - "langchain-text-splitters>=0.3.9", - + "langchain-text-splitters>=0.3.9", + # Class Extraction "pydantic", - + # Developer Specific "ipykernel", - + # Unitree webrtc streaming "aiortc==1.9.0", "pycryptodome", @@ -75,17 +75,17 @@ dependencies = [ # Image "PyTurboJPEG==1.8.2", - + # Audio "openai-whisper", "soundfile", - + # Hugging Face "transformers[torch]==4.49.0", - + # Vector Embedding - "sentence_transformers", - + "sentence_transformers", + # Perception Dependencies "ultralytics>=8.3.70", "filterpy>=1.4.5", @@ -104,7 +104,7 @@ dependencies = [ # Inference "onnx", - # Multiprocess + # Multiprocess "dask[complete]==2025.5.1", # LCM / DimOS utilities @@ -163,7 +163,7 @@ cuda = [ "mmcv>=2.1.0", "xformers>=0.0.20", - # Detic GPU stack + # Detic GPU stack "mss", "dataclasses", "ftfy", @@ -292,6 +292,3 @@ env = [ addopts = "-v -p no:warnings -ra --color=yes -m 'not vis and not benchmark and not exclude and not tool and not needsdata and not lcm and not ros and not heavy and not gpu and not module and not tofix'" asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" - - - diff --git a/requirements.txt b/requirements.txt index 5faa7c8874..808a539624 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ fastapi>=0.115.6 sse-starlette>=2.2.1 uvicorn>=0.34.0 -# Agent Memory +# Agent Memory langchain-chroma>=0.1.4 langchain-openai>=0.2.14 @@ -64,7 +64,7 @@ Pillow mmengine>=0.10.3 mmcv>=2.1.0 timm>=1.0.15 -lap>=0.5.12 +lap>=0.5.12 xformers==0.0.20 # Detic @@ -91,6 +91,6 @@ onnx # Terminal colors rich==14.0.0 -# multiprocess +# multiprocess dask[complete]==2025.5.1 git+https://github.com/dimensionalOS/python_lcm_msgs@main#egg=lcm_msgs diff --git a/tests/__init__.py b/tests/__init__.py index 8b13789179..e69de29bb2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ - diff --git a/tests/isaacsim/run-isaacsim-docker.sh b/tests/isaacsim/run-isaacsim-docker.sh index a9ab642236..2019695960 100644 --- a/tests/isaacsim/run-isaacsim-docker.sh +++ b/tests/isaacsim/run-isaacsim-docker.sh @@ -17,4 +17,4 @@ sudo docker run --network rtsp_net --name isaac-sim --entrypoint bash -it --runt /isaac-sim/python.sh -m pip install -r /dimos/tests/isaacsim/requirements.txt apt-get update apt-get install -y ffmpeg -/isaac-sim/python.sh /dimos/tests/isaacsim/stream_camera.py \ No newline at end of file +/isaac-sim/python.sh /dimos/tests/isaacsim/stream_camera.py diff --git a/tests/isaacsim/setup_ec2.sh b/tests/isaacsim/setup_ec2.sh index 379891e334..f9d33bb3cc 100644 --- a/tests/isaacsim/setup_ec2.sh +++ b/tests/isaacsim/setup_ec2.sh @@ -38,6 +38,5 @@ sudo systemctl restart docker # Verify NVIDIA Container Toolkit sudo docker run --rm --runtime=nvidia --gpus all ubuntu nvidia-smi -# Full isaac sim container +# Full isaac sim container sudo docker pull nvcr.io/nvidia/isaac-sim:4.2.0 - diff --git a/tests/isaacsim/setup_isaacsim_python.sh b/tests/isaacsim/setup_isaacsim_python.sh index 3ed5d8e627..27744482e4 100644 --- a/tests/isaacsim/setup_isaacsim_python.sh +++ b/tests/isaacsim/setup_isaacsim_python.sh @@ -4,11 +4,10 @@ sudo apt install python3.10-venv python3.10 -m venv env_isaacsim source env_isaacsim/bin/activate -# Install pip packages +# Install pip packages pip install isaacsim==4.2.0.2 --extra-index-url https://pypi.nvidia.com pip install isaacsim-extscache-physics==4.2.0.2 pip install isaacsim-extscache-kit==4.2.0.2 pip install isaacsim-extscache-kit-sdk==4.2.0.2 --extra-index-url https://pypi.nvidia.com export OMNI_KIT_ACCEPT_EULA=YES -