diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 860fcd87f2..29ef16fb81 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "dimos-dev", - "image": "ghcr.io/dimensionalos/dev:dev", + "image": "ghcr.io/dimensionalos/ros-dev:dev", "customizations": { "vscode": { "extensions": [ diff --git a/.github/workflows/_docker-build-template.yml b/.github/workflows/_docker-build-template.yml index 11861e8a9b..dd4f3eab66 100644 --- a/.github/workflows/_docker-build-template.yml +++ b/.github/workflows/_docker-build-template.yml @@ -2,9 +2,11 @@ name: docker-build-template on: workflow_call: inputs: - branch-tag: { type: string, required: true } - target: { type: string, required: true } + from-image: { type: string, required: true } + to-image: { type: string, required: true } + dockerfile: { type: string, required: true } freespace: { type: boolean, default: false } + should-run: { type: boolean, default: false } context: { type: string, default: '.' } # you can run this locally as well via @@ -17,6 +19,10 @@ jobs: packages: write steps: + - name: exit early + if: ${{ !inputs.should-run }} + run: | + exit 0 - name: free up disk space # takes a bit of time, so disabled by default # explicitly enable this for large builds @@ -37,34 +43,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Check base image tag - id: tagcheck - env: - BRANCH_TAG: ${{ inputs.branch-tag }} - DOCKERFILE_PATH: docker/${{ inputs.target }}/Dockerfile - DOCKER_CLI_EXPERIMENTAL: enabled # required for `docker buildx imagetools` - run: | - BASE_REPO=$(grep -Eo '^FROM[[:space:]]+ghcr\.io/dimensionalos/[a-zA-Z0-9._-]+' "$DOCKERFILE_PATH" | head -n1 | awk -F/ '{print $NF}') - - if [[ -z "$BASE_REPO" ]]; then - echo "Could not determine base repo from $DOCKERFILE_PATH, reverting to 'dev'" - FROM_TAG="dev" - else - IMAGE="ghcr.io/dimensionalos/${BASE_REPO}:${BRANCH_TAG}" - echo "Checking if $IMAGE exists…" - if docker buildx imagetools inspect "$IMAGE" > /dev/null 2>&1; then - echo "Found $IMAGE" - FROM_TAG="$BRANCH_TAG" - else - echo "Tag '$BRANCH_TAG' not found for $BASE_REPO; falling back to 'dev'" - FROM_TAG="dev" - fi - fi - - echo "from_tag=$FROM_TAG" >> "$GITHUB_OUTPUT" - - - # required for github cache of docker layers - uses: crazy-max/ghaction-github-runtime@v3 @@ -79,9 +57,8 @@ jobs: with: push: true context: ${{ inputs.context }} - file: docker/${{ inputs.target }}/Dockerfile - tags: ghcr.io/dimensionalos/${{ inputs.target }}:${{ inputs.branch-tag }} - cache-from: type=gha,scope=${{ inputs.target }} - cache-to: type=gha,mode=max,scope=${{ inputs.target }} - build-args: | - FROM_TAG=${{ steps.tagcheck.outputs.from_tag }} + file: docker/${{ inputs.dockerfile }}/Dockerfile + tags: ${{ inputs.to-image }} + cache-from: type=gha,scope=${{ inputs.dockerfile }}-${{ inputs.from-image }} + cache-to: type=gha,mode=max,scope=${{ inputs.dockerfile }}-${{ inputs.from-image }} + build-args: FROM_IMAGE=${{ inputs.from-image }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a5500530e3..0de7cc6abe 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -44,7 +44,7 @@ jobs: id: set-tag run: | case "${GITHUB_REF_NAME}" in - master) branch_tag="latest" ;; + main) branch_tag="latest" ;; dev) branch_tag="dev" ;; *) branch_tag=$(echo "${GITHUB_REF_NAME}" \ @@ -56,14 +56,6 @@ jobs: echo "branch tag determined: ${branch_tag}" echo branch_tag="${branch_tag}" >> "$GITHUB_OUTPUT" - ros: - needs: [check-changes] - if: needs.check-changes.outputs.ros == 'true' - uses: ./.github/workflows/_docker-build-template.yml - with: - branch-tag: ${{ needs.check-changes.outputs.branch-tag }} - target: base-ros - # just a debugger inspect-needs: needs: [check-changes, ros] @@ -73,47 +65,105 @@ jobs: - run: | echo '${{ toJSON(needs) }}' - python: + ros: + needs: [check-changes] + if: needs.check-changes.outputs.ros == 'true' + uses: ./.github/workflows/_docker-build-template.yml + with: + from-image: ubuntu:22.04 + to-image: ghcr.io/dimensionalos/ros:${{ needs.check-changes.outputs.branch-tag }} + dockerfile: ros + + # # just a debugger + # inspect-needs: + # needs: [check-changes, ros] + # runs-on: dimos-runner-ubuntu-2204 + # if: always() + # steps: + # - run: | + # echo '${{ toJSON(needs) }}' + + ros-python: needs: [check-changes, ros] - if: | - ${{ - always() && !cancelled() && - needs.check-changes.result == 'success' && - ((needs.ros.result == 'success') || - (needs.ros.result == 'skipped' && - needs.check-changes.outputs.python == 'true')) - }} + if: always() uses: ./.github/workflows/_docker-build-template.yml with: - branch-tag: ${{ needs.check-changes.outputs.branch-tag }} - target: base-ros-python + should-run: ${{ + needs.check-changes.outputs.python == 'true' && + needs.check-changes.result != 'error' && + needs.ros.result != 'error' + }} + + from-image: ghcr.io/dimensionalos/ros:${{ needs.ros.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} + to-image: ghcr.io/dimensionalos/ros-python:${{ needs.check-changes.outputs.branch-tag }} + dockerfile: python freespace: true + python: + needs: [check-changes] + if: needs.check-changes.outputs.python == 'true' + uses: ./.github/workflows/_docker-build-template.yml + with: + should-run: true + freespace: true + dockerfile: python + from-image: ubuntu:22.04 + to-image: ghcr.io/dimensionalos/python:${{ needs.check-changes.outputs.branch-tag }} + dev: needs: [check-changes, python] - if: | - ${{ - always() && !cancelled() && - needs.check-changes.result == 'success' && - ((needs.python.result == 'success') || - (needs.python.result == 'skipped' && - needs.check-changes.outputs.dev == 'true')) - }} + if: always() + + uses: ./.github/workflows/_docker-build-template.yml + with: + should-run: ${{ + needs.check-changes.result == 'success' && + ((needs.python.result == 'success') || + (needs.python.result == 'skipped' && + needs.check-changes.outputs.dev == 'true')) }} + from-image: ghcr.io/dimensionalos/python:${{ needs.python.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} + to-image: ghcr.io/dimensionalos/dev:${{ needs.check-changes.outputs.branch-tag }} + dockerfile: dev + + ros-dev: + needs: [check-changes, ros-python] + if: always() uses: ./.github/workflows/_docker-build-template.yml with: - branch-tag: ${{ needs.check-changes.outputs.branch-tag }} - target: dev + should-run: ${{ + needs.check-changes.result == 'success' && + ((needs.ros-python.result == 'success') || + (needs.ros-python.result == 'skipped' && + needs.check-changes.outputs.dev == 'true')) + }} + from-image: ghcr.io/dimensionalos/ros-python:${{ needs.ros-python.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} + to-image: ghcr.io/dimensionalos/ros-dev:${{ needs.check-changes.outputs.branch-tag }} + dockerfile: dev + + run-ros-tests: + needs: [check-changes, ros-dev] + if: always() + uses: ./.github/workflows/tests.yml + with: + should-run: ${{ + needs.check-changes.result == 'success' && + ((needs.ros-dev.result == 'success') || + (needs.ros-dev.result == 'skipped' && + needs.check-changes.outputs.tests == 'true')) + }} + cmd: "pytest && pytest -m ros" # run tests that depend on ros as well + dev-image: ros-dev:${{ needs.ros-dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} run-tests: needs: [check-changes, dev] - if: | - ${{ - always() && !cancelled() && - needs.check-changes.result == 'success' && - ((needs.dev.result == 'success') || - (needs.dev.result == 'skipped' && - needs.check-changes.outputs.tests == 'true')) - }} + if: always() uses: ./.github/workflows/tests.yml with: - branch-tag: ${{ needs.dev.result != 'success' && 'dev' || needs.check-changes.outputs.branch-tag }} + should-run: ${{ + needs.check-changes.result == 'success' && + ((needs.dev.result == 'success') || + (needs.dev.result == 'skipped' && + needs.check-changes.outputs.tests == 'true')) + }} + cmd: "pytest" + dev-image: dev:${{ needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7efc7bad01..a9cdb78abf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,10 +3,17 @@ name: tests on: workflow_call: inputs: - branch-tag: + should-run: + required: false + type: boolean + default: true + dev-image: + required: true + type: string + default: "dev:dev" + cmd: required: true type: string - default: "latest" permissions: contents: read @@ -17,12 +24,17 @@ jobs: runs-on: dimos-runner-ubuntu-2204 container: - image: ghcr.io/dimensionalos/dev:${{ inputs.branch-tag }} + image: ghcr.io/dimensionalos/${{ inputs.dev-image }} steps: + - name: exit early + if: ${{ !inputs.should-run }} + run: | + exit 0 + - uses: actions/checkout@v4 - name: Run tests run: | git config --global --add safe.directory '*' - /entrypoint.sh bash -c "pytest" + /entrypoint.sh bash -c "${{ inputs.cmd }}" diff --git a/dimos/robot/local_planner/local_planner.py b/dimos/robot/local_planner/local_planner.py index 4bd0b4a790..2c559afae6 100644 --- a/dimos/robot/local_planner/local_planner.py +++ b/dimos/robot/local_planner/local_planner.py @@ -30,7 +30,7 @@ from dimos.types.vector import VectorLike, Vector, to_tuple from dimos.types.path import Path -from nav_msgs.msg import OccupancyGrid +from dimos.types.costmap import Costmap logger = setup_logger("dimos.robot.unitree.local_planner", level=logging.DEBUG) @@ -45,7 +45,7 @@ class BaseLocalPlanner(ABC): def __init__( self, - get_costmap: Callable[[], Optional[OccupancyGrid]], + get_costmap: Callable[[], Optional[Costmap]], transform: object, move_vel_control: Callable[[float, float, float], None], safety_threshold: float = 0.5, diff --git a/dimos/robot/test_ros_observable_topic.py b/dimos/robot/test_ros_observable_topic.py index 4df9dbb0de..71a1484de3 100644 --- a/dimos/robot/test_ros_observable_topic.py +++ b/dimos/robot/test_ros_observable_topic.py @@ -15,9 +15,7 @@ import threading import time -from nav_msgs import msg import pytest -from dimos.robot.ros_observable_topic import ROSObservableTopicAbility from dimos.utils.logging_config import setup_logger from dimos.types.vector import Vector import asyncio @@ -68,11 +66,18 @@ def destroy_subscription(self, subscription): self.logger.info(f"Unknown subscription: {subscription}") -class MockRobot(ROSObservableTopicAbility): - def __init__(self): - self.logger = setup_logger("ROBOT") - # Initialize the mock ROS node - self._node = MockROSNode() +# we are doing this in order to avoid importing ROS dependencies if ros tests aren't runnin +@pytest.fixture +def robot(): + from dimos.robot.ros_observable_topic import ROSObservableTopicAbility + + class MockRobot(ROSObservableTopicAbility): + def __init__(self): + self.logger = setup_logger("ROBOT") + # Initialize the mock ROS node + self._node = MockROSNode() + + return MockRobot() # This test verifies a bunch of basics: @@ -82,8 +87,10 @@ def __init__(self): # 3. that the system unsubscribes from ROS when observers are disposed # 4. that the system replays the last message to new observers, # before the new ROS sub starts producing -def test_parallel_and_cleanup(): - robot = MockRobot() +@pytest.mark.ros +def test_parallel_and_cleanup(robot): + from nav_msgs import msg + received_messages = [] obs1 = robot.topic("/odom", msg.Odometry) @@ -152,8 +159,9 @@ def test_parallel_and_cleanup(): # ROS thread ─► ReplaySubject─► observe_on(pool) ─► backpressure.latest ─► sub1 (fast) # ├──► observe_on(pool) ─► backpressure.latest ─► sub2 (slow) # └──► observe_on(pool) ─► backpressure.latest ─► sub3 (slower) -def test_parallel_and_hog(): - robot = MockRobot() +@pytest.mark.ros +def test_parallel_and_hog(robot): + from nav_msgs import msg obs1 = robot.topic("/odom", msg.Odometry) obs2 = robot.topic("/odom", msg.Odometry) @@ -191,8 +199,9 @@ def test_parallel_and_hog(): @pytest.mark.asyncio -async def test_topic_latest_async(): - robot = MockRobot() +@pytest.mark.ros +async def test_topic_latest_async(robot): + from nav_msgs import msg odom = await robot.topic_latest_async("/odom", msg.Odometry) assert odom() == 1 @@ -203,15 +212,16 @@ async def test_topic_latest_async(): assert robot._node.subs == {} -def test_topic_auto_conversion(): - robot = MockRobot() +@pytest.mark.ros +def test_topic_auto_conversion(robot): odom = robot.topic("/vector", Vector).subscribe(lambda x: print(x)) time.sleep(0.5) odom.dispose() -def test_topic_latest_sync(): - robot = MockRobot() +@pytest.mark.ros +def test_topic_latest_sync(robot): + from nav_msgs import msg odom = robot.topic_latest("/odom", msg.Odometry) assert odom() == 1 @@ -222,8 +232,9 @@ def test_topic_latest_sync(): assert robot._node.subs == {} -def test_topic_latest_sync_benchmark(): - robot = MockRobot() +@pytest.mark.ros +def test_topic_latest_sync_benchmark(robot): + from nav_msgs import msg odom = robot.topic_latest("/odom", msg.Odometry) @@ -242,10 +253,3 @@ def test_topic_latest_sync_benchmark(): odom.dispose() time.sleep(0.1) assert robot._node.subs == {} - - -if __name__ == "__main__": - test_parallel_and_cleanup() - test_parallel_and_hog() - test_topic_latest_sync() - asyncio.run(test_topic_latest_async()) diff --git a/dimos/robot/unitree_webrtc/type/test_odometry.py b/dimos/robot/unitree_webrtc/type/test_odometry.py index 74fa512177..2e3ee9758e 100644 --- a/dimos/robot/unitree_webrtc/type/test_odometry.py +++ b/dimos/robot/unitree_webrtc/type/test_odometry.py @@ -14,19 +14,16 @@ from __future__ import annotations -from functools import reduce -import math from operator import sub, add import os import threading -from typing import Iterable, Optional +from typing import Optional import reactivex.operators as ops import pytest from dotenv import load_dotenv -from dimos.robot.unitree_webrtc.type.odometry import Odometry, RawOdometryMessage -from dimos.robot.unitree_webrtc.unitree_go2 import UnitreeGo2 +from dimos.robot.unitree_webrtc.type.odometry import Odometry from dimos.utils.testing import SensorReplay, SensorStorage _EXPECTED_TOTAL_RAD = -4.05212 @@ -91,6 +88,8 @@ def test_total_rotation_travel_rxpy() -> None: # data collection tool @pytest.mark.tool def test_store_odometry_stream() -> None: + from dimos.robot.unitree_webrtc.unitree_go2 import UnitreeGo2 + load_dotenv() robot = UnitreeGo2(ip=os.getenv("ROBOT_IP"), mode="ai") diff --git a/dimos/types/costmap.py b/dimos/types/costmap.py index b349af7fd5..1107abf8bf 100644 --- a/dimos/types/costmap.py +++ b/dimos/types/costmap.py @@ -17,7 +17,7 @@ import numpy as np from typing import Optional from scipy import ndimage -from nav_msgs.msg import OccupancyGrid +from dimos.types.ros_polyfill import OccupancyGrid from dimos.types.vector import Vector, VectorLike, x, y, to_vector import open3d as o3d diff --git a/dimos/types/ros_polyfill.py b/dimos/types/ros_polyfill.py new file mode 100644 index 0000000000..b5c2bc1d64 --- /dev/null +++ b/dimos/types/ros_polyfill.py @@ -0,0 +1,103 @@ +# Copyright 2025 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + from geometry_msgs.msg import Vector3 +except ImportError: + + class Vector3: + def __init__(self, x: float = 0.0, y: float = 0.0, z: float = 0.0): + self.x = float(x) + self.y = float(y) + self.z = float(z) + + def __repr__(self) -> str: + return f"Vector3(x={self.x}, y={self.y}, z={self.z})" + + +try: + from nav_msgs.msg import OccupancyGrid, Odometry + from geometry_msgs.msg import Pose, Point, Quaternion, Twist + from std_msgs.msg import Header +except ImportError: + + class Header: + def __init__(self): + self.stamp = None + self.frame_id = "" + + class Point: + def __init__(self, x: float = 0.0, y: float = 0.0, z: float = 0.0): + self.x = float(x) + self.y = float(y) + self.z = float(z) + + def __repr__(self) -> str: + return f"Point(x={self.x}, y={self.y}, z={self.z})" + + class Quaternion: + def __init__(self, x: float = 0.0, y: float = 0.0, z: float = 0.0, w: float = 1.0): + self.x = float(x) + self.y = float(y) + self.z = float(z) + self.w = float(w) + + def __repr__(self) -> str: + return f"Quaternion(x={self.x}, y={self.y}, z={self.z}, w={self.w})" + + class Pose: + def __init__(self): + self.position = Point() + self.orientation = Quaternion() + + def __repr__(self) -> str: + return f"Pose(position={self.position}, orientation={self.orientation})" + + class MapMetaData: + def __init__(self): + self.map_load_time = None + self.resolution = 0.05 + self.width = 0 + self.height = 0 + self.origin = Pose() + + def __repr__(self) -> str: + return f"MapMetaData(resolution={self.resolution}, width={self.width}, height={self.height}, origin={self.origin})" + + class Twist: + def __init__(self): + self.linear = Vector3() + self.angular = Vector3() + + def __repr__(self) -> str: + return f"Twist(linear={self.linear}, angular={self.angular})" + + class OccupancyGrid: + def __init__(self): + self.header = Header() + self.info = MapMetaData() + self.data = [] + + def __repr__(self) -> str: + return f"OccupancyGrid(info={self.info}, data_length={len(self.data)})" + + class Odometry: + def __init__(self): + self.header = Header() + self.child_frame_id = "" + self.pose = Pose() + self.twist = Twist() + + def __repr__(self) -> str: + return f"Odometry(pose={self.pose}, twist={self.twist})" diff --git a/dimos/types/vector.py b/dimos/types/vector.py index 9b01c6a26a..d980e28105 100644 --- a/dimos/types/vector.py +++ b/dimos/types/vector.py @@ -15,7 +15,7 @@ from typing import List, Tuple, TypeVar, Union, Sequence import numpy as np -from geometry_msgs.msg import Vector3 +from dimos.types.ros_polyfill import Vector3 T = TypeVar("T", bound="Vector") diff --git a/docker/base-ros-python/Dockerfile b/docker/base-ros-python/Dockerfile deleted file mode 100644 index f86974b41a..0000000000 --- a/docker/base-ros-python/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# trigger rebuild: 1 -ARG FROM_TAG=latest -FROM ghcr.io/dimensionalos/base-ros:${FROM_TAG} - -RUN mkdir -p /app/dimos - -COPY requirements.txt /app/ -COPY base-requirements.txt /app/ - -WORKDIR /app - -RUN --mount=type=cache,target=/root/.cache/pip pip install -r base-requirements.txt - -RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index a210fd4e15..2c8c828059 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,5 +1,5 @@ -ARG FROM_TAG=latest -FROM ghcr.io/dimensionalos/base-ros-python:${FROM_TAG} +ARG FROM_IMAGE=ghcr.io/dimensionalos/ros-python:dev +FROM ${FROM_IMAGE} ARG GIT_COMMIT=unknown ARG GIT_BRANCH=unknown @@ -14,7 +14,9 @@ RUN apt-get install -y \ htop \ python-is-python3 \ iputils-ping \ - wget + wget \ + pre-commit + # Configure git to trust any directory (resolves dubious ownership issues in containers) RUN git config --global --add safe.directory '*' @@ -27,6 +29,12 @@ COPY motd /etc/motd COPY /docker/dev/bash.sh /root/.bash.sh COPY /docker/dev/tmux.conf /root/.tmux.conf +# Install nodejs (for random devtooling like copilot etc) +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash +ENV NVM_DIR=/root/.nvm +RUN bash -c "source $NVM_DIR/nvm.sh && nvm install 24" + +# This doesn't work atm RUN echo " v_${GIT_BRANCH}:${GIT_COMMIT} | $(date)" >> /etc/motd RUN echo "echo -e '\033[34m$(cat /etc/motd)\033[0m\n'" >> /root/.bashrc diff --git a/docker/dev/entrypoint.sh b/docker/dev/entrypoint.sh index fb78050376..d48bea16e3 100644 --- a/docker/dev/entrypoint.sh +++ b/docker/dev/entrypoint.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash -source /opt/ros/${ROS_DISTRO}/setup.bash -#source /ros2_ws/install/setup.bash +if [ -d "/opt/ros/${ROS_DISTRO}" ]; then + source /opt/ros/${ROS_DISTRO}/setup.bash +else + echo "ROS is not available in this env" +fi exec "$@" diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile new file mode 100644 index 0000000000..f08510faa5 --- /dev/null +++ b/docker/python/Dockerfile @@ -0,0 +1,43 @@ +ARG FROM_IMAGE=ghcr.io/dimensionalos/ros:dev +FROM ${FROM_IMAGE} + +# Install basic requirements +RUN apt-get update +RUN apt-get install -y \ + curl \ + gnupg2 \ + lsb-release \ + python3-pip \ + clang \ + portaudio19-dev \ + git \ + mesa-utils \ + libgl1-mesa-glx \ + libgl1-mesa-dri \ + software-properties-common \ + libxcb1-dev \ + libxcb-keysyms1-dev \ + libxcb-util0-dev \ + libxcb-icccm4-dev \ + libxcb-image0-dev \ + libxcb-randr0-dev \ + libxcb-shape0-dev \ + libxcb-xinerama0-dev \ + libxcb-xkb-dev \ + libxkbcommon-x11-dev \ + qtbase5-dev \ + qtchooser \ + qt5-qmake \ + qtbase5-dev-tools \ + supervisor + +RUN mkdir -p /app/dimos + +COPY requirements.txt /app/ +COPY base-requirements.txt /app/ + +WORKDIR /app + +RUN --mount=type=cache,target=/root/.cache/pip pip install -r base-requirements.txt + +RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt diff --git a/docker/base-ros/Dockerfile b/docker/ros/Dockerfile similarity index 98% rename from docker/base-ros/Dockerfile rename to docker/ros/Dockerfile index b1814ba54b..df6544f23b 100644 --- a/docker/base-ros/Dockerfile +++ b/docker/ros/Dockerfile @@ -1,5 +1,5 @@ -# trigger rebuild: 1 -FROM ubuntu:22.04 +ARG FROM_IMAGE=ubuntu:22.04 +FROM ${FROM_IMAGE} # Avoid prompts from apt ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/base-ros/install-nix.sh b/docker/ros/install-nix.sh similarity index 100% rename from docker/base-ros/install-nix.sh rename to docker/ros/install-nix.sh diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 0000000000..a041ab08cc --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,146 @@ +# Continuous Integration Guide + +> *If you are ******not****** editing CI-related files, you can safely ignore this document.* + +Our GitHub Actions pipeline lives in **`.github/workflows/`** and is split into three top-level workflows: + +| Workflow | File | Purpose | +| ----------- | ------------- | -------------------------------------------------------------------- | +| **cleanup** | `cleanup.yml` | Auto-formats code with *pre-commit* and pushes fixes to your branch. | +| **docker** | `docker.yml` | Builds (and caches) our Docker image hierarchy. | +| **tests** | `tests.yml` | Pulls the *dev* image and runs the test suite. | + +--- + +## `cleanup.yml` + +* Checks out the branch. +* Executes **pre-commit** hooks. +* If hooks modify files, commits and pushes the changes back to the same branch. + +> This guarantees consistent formatting even if the developer has not installed pre-commit locally. + +--- + +## `tests.yml` + +* Pulls the pre-built **dev** container image. +* Executes: + +```bash +pytest +``` + +That’s it—making the job trivial to reproduce locally via: + +```bash +./bin/dev # enter container +pytest # run tests +``` + +--- + +## `docker.yml` + +### Objectives + +1. **Layered images**: each image builds on its parent, enabling parallel builds once dependencies are ready. +2. **Speed**: build children as soon as parents finish; leverage aggressive caching. +3. **Minimal work**: skip images whose context hasn’t changed. + +### Current hierarchy + + +``` + ┌──────┐ + │ubuntu│ + └┬────┬┘ + ┌▽──┐┌▽───────┐ + │ros││python │ + └┬──┘└───────┬┘ + ┌▽─────────┐┌▽──┐ + │ros-python││dev│ + └┬─────────┘└───┘ + ┌▽──────┐ + │ros-dev│ + └───────┘ +``` + +* ghcr.io/dimensionalos/ros:dev +* ghcr.io/dimensionalos/python:dev +* ghcr.io/dimensionalos/ros-python:dev +* ghcr.io/dimensionalos/ros-dev:dev +* ghcr.io/dimensionalos/dev:dev + +> **Note**: The diagram shows only currently active images; the system is extensible—new combinations are possible, builds can be run per branch and as parallel as possible + + +``` + ┌──────┐ + │ubuntu│ + └┬────┬┘ + ┌▽──┐┌▽────────────────────────┐ + │ros││python │ + └┬──┘└───────────────────┬────┬┘ + ┌▽─────────────────────┐┌▽──┐┌▽──────┐ + │ros-python ││dev││unitree│ + └┬────────┬───────────┬┘└───┘└───────┘ + ┌▽──────┐┌▽─────────┐┌▽──────────┐ + │ros-dev││ros-jetson││ros-unitree│ + └───────┘└──────────┘└───────────┘ +``` + +### Branch-aware tagging + +When a branch triggers a build: + +* Only images whose context changed are rebuilt. +* New images receive the tag `:`. +* Unchanged parents are pulled from the registry, e.g. + +given we made python requirements.txt changes, but no ros changes, image dep graph would look like this: + +``` +ghcr.io/dimensionalos/ros:dev → ghcr.io/dimensionalos/ros-python:my_branch → ghcr.io/dimensionalos/dev:my_branch +``` + +### Job matrix & the **check-changes** step + +To decide what to build we run a `check-changes` job that compares the diff against path filters: + +```yaml +filters: | + ros: + - .github/workflows/_docker-build-template.yml + - .github/workflows/docker.yml + - docker/base-ros/** + + python: + - docker/base-python/** + - requirements*.txt + + dev: + - docker/dev/** +``` + +This populates a build matrix (ros, python, dev) with `true/false` flags. + +### The dependency execution issue + +Ideally a child job (e.g. **ros-python**) should depend on both: + +* **check-changes** (to know if it *should* run) +* Its **parent image job** (to wait for the artifact) + +GitHub Actions can’t express “run only if *both* conditions are true *and* the parent job wasn’t skipped”. + +We are using `needs: [check-changes, ros]` to ensure the job runs after the ros build, but if ros build has been skipped we need `if: always()` to ensure that the build runs anyway. +Adding `always` for some reason completely breaks the conditional check, we cannot have OR, AND operators, it just makes the job _always_ run, which means we build python even if we don't need to. + +This is unfortunate as the build takes ~30 min first time (a few minutes afterwards thanks to caching) and I've spent a lot of time on this, lots of viable seeming options didn't pan out and probably we need to completely rewrite and own the actions runner and not depend on github structure at all. Single job called `CI` or something, within our custom docker image. + +--- + +## `run-tests` (job inside `docker.yml`) + +After all requested images are built, this job triggers **tests.yml**, passing the freshly created *dev* image tag so the suite runs against the branch-specific environment. diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..e6d920a293 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1748929857, + "narHash": "sha256-lcZQ8RhsmhsK8u7LIFsJhsLh/pzR9yZ8yqpTzyGdj+Q=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c2a03962b8e24e669fb37b7df10e7c79531ff1a4", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..60751b156f --- /dev/null +++ b/flake.nix @@ -0,0 +1,95 @@ +{ + description = "Project dev environment as Nix shell + DockerTools layered image"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils, ... }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { inherit system; }; + + # ------------------------------------------------------------ + # 1. Shared package list (tool-chain + project deps) + # ------------------------------------------------------------ + devPackages = with pkgs; [ + ### Core shell & utils + bashInteractive coreutils gh + stdenv.cc.cc.lib + + ### Python + static analysis + python312 python312Packages.pip python312Packages.setuptools + python312Packages.virtualenv ruff mypy pre-commit + + ### Runtime deps + python312Packages.pyaudio portaudio ffmpeg_6 ffmpeg_6.dev + + ### Graphics / X11 stack + libGL libGLU mesa glfw + xorg.libX11 xorg.libXi xorg.libXext xorg.libXrandr xorg.libXinerama + xorg.libXcursor xorg.libXfixes xorg.libXrender xorg.libXdamage + xorg.libXcomposite xorg.libxcb xorg.libXScrnSaver xorg.libXxf86vm + + udev SDL2 SDL2.dev zlib + + ### GTK / OpenCV helpers + glib gtk3 gdk-pixbuf gobject-introspection + + ### Open3D & build-time + eigen cmake ninja jsoncpp libjpeg libpng + ]; + + # ------------------------------------------------------------ + # 2. Host interactive shell → `nix develop` + # ------------------------------------------------------------ + devShell = pkgs.mkShell { + packages = devPackages; + shellHook = '' + export LD_LIBRARY_PATH="${pkgs.lib.makeLibraryPath [ + pkgs.stdenv.cc.cc.lib pkgs.libGL pkgs.libGLU pkgs.mesa pkgs.glfw + pkgs.xorg.libX11 pkgs.xorg.libXi pkgs.xorg.libXext pkgs.xorg.libXrandr + pkgs.xorg.libXinerama pkgs.xorg.libXcursor pkgs.xorg.libXfixes + pkgs.xorg.libXrender pkgs.xorg.libXdamage pkgs.xorg.libXcomposite + pkgs.xorg.libxcb pkgs.xorg.libXScrnSaver pkgs.xorg.libXxf86vm + pkgs.udev pkgs.portaudio pkgs.SDL2.dev pkgs.zlib pkgs.glib pkgs.gtk3 + pkgs.gdk-pixbuf pkgs.gobject-introspection]}:$LD_LIBRARY_PATH" + + export DISPLAY=:0 + + PROJECT_ROOT=$(git rev-parse --show-toplevel 2>/dev/null || echo "$PWD") + if [ -f "$PROJECT_ROOT/env/bin/activate" ]; then + . "$PROJECT_ROOT/env/bin/activate" + fi + + [ -f "$PROJECT_ROOT/motd" ] && cat "$PROJECT_ROOT/motd" + [ -f "$PROJECT_ROOT/.pre-commit-config.yaml" ] && pre-commit install --install-hooks + ''; + }; + + # ------------------------------------------------------------ + # 3. Closure copied into the OCI image rootfs + # ------------------------------------------------------------ + imageRoot = pkgs.buildEnv { + name = "dimos-image-root"; + paths = devPackages; + pathsToLink = [ "/bin" ]; + }; + + in { + ## Local dev shell + devShells.default = devShell; + + ## Layered docker image with DockerTools + packages.devcontainer = pkgs.dockerTools.buildLayeredImage { + name = "dimensionalos/dimos-dev"; + tag = "latest"; + contents = [ imageRoot ]; + config = { + WorkingDir = "/workspace"; + Cmd = [ "bash" ]; + }; + }; + }); +} diff --git a/pyproject.toml b/pyproject.toml index bbee605a7f..2f81fd62b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ markers = [ "benchmark: benchmark, executes something multiple times, calculates avg, prints to console", "exclude: arbitrary exclusion from CI and default test exec", "tool: dev tooling", - "needsdata: needs test data to be downloaded"] + "needsdata: needs test data to be downloaded", + "ros: depend on ros"] -addopts = "-v -ra --color=yes -m 'not vis and not benchmark and not exclude and not tool and not needsdata'" +addopts = "-v -ra --color=yes -m 'not vis and not benchmark and not exclude and not tool and not needsdata and not ros'" diff --git a/requirements.txt b/requirements.txt index 79b6393265..7f71ec17cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -94,3 +94,5 @@ git+https://github.com/facebookresearch/detectron2.git@v0.6 # Mapping open3d + +# Touch for rebuild