From 78da7b38117c339d4081960b65fcfc5daf89bf94 Mon Sep 17 00:00:00 2001 From: lesh Date: Wed, 4 Jun 2025 17:00:49 +0300 Subject: [PATCH 01/30] ci docs --- docs/ci.md | 137 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 docs/ci.md diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 0000000000..ee4182e8ea --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,137 @@ +# Continuous Integration Guide + +> *If you are ******not****** editing CI-related files, you can safely ignore this document.* + +Our GitHub Actions pipeline lives in **`.github/workflows/`** and is split into three top-level workflows: + +| Workflow | File | Purpose | +| ----------- | ------------- | -------------------------------------------------------------------- | +| **cleanup** | `cleanup.yml` | Auto-formats code with *pre-commit* and pushes fixes to your branch. | +| **docker** | `docker.yml` | Builds (and caches) our Docker image hierarchy. | +| **tests** | `tests.yml` | Pulls the *dev* image and runs the test suite. | + +--- + +## `cleanup.yml` + +* Checks out the branch. +* Executes **pre-commit** hooks. +* If hooks modify files, commits and pushes the changes back to the same branch. + +> This guarantees consistent formatting even if the developer has not installed pre-commit locally. + +--- + +## `tests.yml` + +* Pulls the pre-built **dev** container image. +* Executes: + +```bash +pytest +``` + +That’s it—making the job trivial to reproduce locally via: + +```bash +./bin/dev # enter container +pytest # run tests +``` + +--- + +## `docker.yml` + +### Objectives + +1. **Layered images**: each image builds on its parent, enabling parallel builds once dependencies are ready. +2. **Speed**: build children as soon as parents finish; leverage aggressive caching. +3. **Minimal work**: skip images whose context hasn’t changed. + +### Current hierarchy + + +``` + ┌───┐ + │ros│ + └┬──┘ + ┌▽─────┐ + │python│ + └┬─────┘ + ┌▽──┐ + │dev│ + └───┘ +``` + +> **Note**: The diagram shows only currently active images; the system is extensible—new combinations are possible, builds can be run per branch and as parallel as possible + + +``` + ┌──────┐ + │ubuntu│ + └┬────┬┘ + ┌▽──┐┌▽────────────────────────┐ + │ros││python │ + └┬──┘└───────────────────┬────┬┘ + ┌▽─────────────────────┐┌▽──┐┌▽──────┐ + │ros-python ││dev││unitree│ + └┬────────┬───────────┬┘└───┘└───────┘ + ┌▽──────┐┌▽─────────┐┌▽──────────┐ + │ros-dev││ros-jetson││ros-unitree│ + └───────┘└──────────┘└───────────┘ +``` + +### Branch-aware tagging + +When a branch triggers a build: + +* Only images whose context changed are rebuilt. +* New images receive the tag `:`. +* Unchanged parents are pulled from the registry, e.g. + +given we made python requirements.txt changes, but no ros changes, image dep graph would look like this: + +``` +ghcr.io/dimensionalos/ros:dev → ghcr.io/dimensionalos/ros-python:my_branch → ghcr.io/dimensionalos/dev:my_branch +``` + +### Job matrix & the **check-changes** step + +To decide what to build we run a `check-changes` job that compares the diff against path filters: + +```yaml +filters: | + ros: + - .github/workflows/_docker-build-template.yml + - .github/workflows/docker.yml + - docker/base-ros/** + + python: + - docker/base-python/** + - requirements*.txt + + dev: + - docker/dev/** +``` + +This populates a build matrix (ros, python, dev) with `true/false` flags. + +### The dependency execution issue + +Ideally a child job (e.g. **ros-python**) should depend on both: + +* **check-changes** (to know if it *should* run) +* Its **parent image job** (to wait for the artifact) + +GitHub Actions can’t express “run only if *both* conditions are true *and* the parent job wasn’t skipped”. + +We are using `needs: [check-changes, ros]` to ensure the job runs after the ros build, but if ros build has been skipped we need `if: always()` to ensure that the build runs anyway. +Adding `always` for some reason completely breaks the conditional check, we cannot have OR, AND operators, it just makes the job _always_ run, which means we build python even if we don't need to. + +This is unfortunate as the build takes ~30 min first time (a few minutes afterwards thanks to caching) and I've spent a lot of time on this, lots of viable seeming options didn't pan out and probably we need to completely rewrite and own the actions runner and not depend on github structure at all. Single job called `CI` or something, within our custom docker image. + +--- + +## `run-tests` (job inside `docker.yml`) + +After all requested images are built, this job triggers **tests.yml**, passing the freshly created *dev* image tag so the suite runs against the branch-specific environment. From d2ea7375bd419789648efafa767c060ee048148a Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 13:12:48 +0300 Subject: [PATCH 02/30] implemented ros polyfill for vector --- dimos/types/vector.py | 2 +- flake.lock | 61 +++++++++++++++++++++++++++ flake.nix | 95 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/dimos/types/vector.py b/dimos/types/vector.py index 9b01c6a26a..d980e28105 100644 --- a/dimos/types/vector.py +++ b/dimos/types/vector.py @@ -15,7 +15,7 @@ from typing import List, Tuple, TypeVar, Union, Sequence import numpy as np -from geometry_msgs.msg import Vector3 +from dimos.types.ros_polyfill import Vector3 T = TypeVar("T", bound="Vector") diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..e6d920a293 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1748929857, + "narHash": "sha256-lcZQ8RhsmhsK8u7LIFsJhsLh/pzR9yZ8yqpTzyGdj+Q=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c2a03962b8e24e669fb37b7df10e7c79531ff1a4", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..60751b156f --- /dev/null +++ b/flake.nix @@ -0,0 +1,95 @@ +{ + description = "Project dev environment as Nix shell + DockerTools layered image"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils, ... }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { inherit system; }; + + # ------------------------------------------------------------ + # 1. Shared package list (tool-chain + project deps) + # ------------------------------------------------------------ + devPackages = with pkgs; [ + ### Core shell & utils + bashInteractive coreutils gh + stdenv.cc.cc.lib + + ### Python + static analysis + python312 python312Packages.pip python312Packages.setuptools + python312Packages.virtualenv ruff mypy pre-commit + + ### Runtime deps + python312Packages.pyaudio portaudio ffmpeg_6 ffmpeg_6.dev + + ### Graphics / X11 stack + libGL libGLU mesa glfw + xorg.libX11 xorg.libXi xorg.libXext xorg.libXrandr xorg.libXinerama + xorg.libXcursor xorg.libXfixes xorg.libXrender xorg.libXdamage + xorg.libXcomposite xorg.libxcb xorg.libXScrnSaver xorg.libXxf86vm + + udev SDL2 SDL2.dev zlib + + ### GTK / OpenCV helpers + glib gtk3 gdk-pixbuf gobject-introspection + + ### Open3D & build-time + eigen cmake ninja jsoncpp libjpeg libpng + ]; + + # ------------------------------------------------------------ + # 2. Host interactive shell → `nix develop` + # ------------------------------------------------------------ + devShell = pkgs.mkShell { + packages = devPackages; + shellHook = '' + export LD_LIBRARY_PATH="${pkgs.lib.makeLibraryPath [ + pkgs.stdenv.cc.cc.lib pkgs.libGL pkgs.libGLU pkgs.mesa pkgs.glfw + pkgs.xorg.libX11 pkgs.xorg.libXi pkgs.xorg.libXext pkgs.xorg.libXrandr + pkgs.xorg.libXinerama pkgs.xorg.libXcursor pkgs.xorg.libXfixes + pkgs.xorg.libXrender pkgs.xorg.libXdamage pkgs.xorg.libXcomposite + pkgs.xorg.libxcb pkgs.xorg.libXScrnSaver pkgs.xorg.libXxf86vm + pkgs.udev pkgs.portaudio pkgs.SDL2.dev pkgs.zlib pkgs.glib pkgs.gtk3 + pkgs.gdk-pixbuf pkgs.gobject-introspection]}:$LD_LIBRARY_PATH" + + export DISPLAY=:0 + + PROJECT_ROOT=$(git rev-parse --show-toplevel 2>/dev/null || echo "$PWD") + if [ -f "$PROJECT_ROOT/env/bin/activate" ]; then + . "$PROJECT_ROOT/env/bin/activate" + fi + + [ -f "$PROJECT_ROOT/motd" ] && cat "$PROJECT_ROOT/motd" + [ -f "$PROJECT_ROOT/.pre-commit-config.yaml" ] && pre-commit install --install-hooks + ''; + }; + + # ------------------------------------------------------------ + # 3. Closure copied into the OCI image rootfs + # ------------------------------------------------------------ + imageRoot = pkgs.buildEnv { + name = "dimos-image-root"; + paths = devPackages; + pathsToLink = [ "/bin" ]; + }; + + in { + ## Local dev shell + devShells.default = devShell; + + ## Layered docker image with DockerTools + packages.devcontainer = pkgs.dockerTools.buildLayeredImage { + name = "dimensionalos/dimos-dev"; + tag = "latest"; + contents = [ imageRoot ]; + config = { + WorkingDir = "/workspace"; + Cmd = [ "bash" ]; + }; + }; + }); +} From c03a810e88c46872d9a416ab7407e56096bcdd32 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 13:12:59 +0300 Subject: [PATCH 03/30] added no ros docker builds to workflow --- .github/workflows/docker.yml | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a5500530e3..3bbb154771 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -73,7 +73,7 @@ jobs: - run: | echo '${{ toJSON(needs) }}' - python: + ros-python: needs: [check-changes, ros] if: | ${{ @@ -89,8 +89,17 @@ jobs: target: base-ros-python freespace: true - dev: - needs: [check-changes, python] + python: + needs: [check-changes] + if: needs.check-changes.outputs.python == 'true' + uses: ./.github/workflows/_docker-build-template.yml + with: + branch-tag: ${{ needs.check-changes.outputs.branch-tag }} + target: base-python + freespace: true + + noros-dev: + needs: [check-changes, ros-python] if: | ${{ always() && !cancelled() && @@ -100,6 +109,21 @@ jobs: needs.check-changes.outputs.dev == 'true')) }} uses: ./.github/workflows/_docker-build-template.yml + with: + branch-tag: ${{ needs.check-changes.outputs.branch-tag }} + target: noros-dev + + dev: + needs: [check-changes, ros-python] + if: | + ${{ + always() && !cancelled() && + needs.check-changes.result == 'success' && + ((needs.ros-python.result == 'success') || + (needs.ros-python.result == 'skipped' && + needs.check-changes.outputs.dev == 'true')) + }} + uses: ./.github/workflows/_docker-build-template.yml with: branch-tag: ${{ needs.check-changes.outputs.branch-tag }} target: dev From e79f16cffbee419709c6c78436bc0942e0deeb0f Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 13:15:44 +0300 Subject: [PATCH 04/30] triggering base py rebuild --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 79b6393265..7f71ec17cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -94,3 +94,5 @@ git+https://github.com/facebookresearch/detectron2.git@v0.6 # Mapping open3d + +# Touch for rebuild From 07c124ab0a2b8c8bc1e147ebf603f7ebb90c791e Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 13:16:54 +0300 Subject: [PATCH 05/30] noros dev uncoupled from ros-python --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3bbb154771..fe425022d0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -99,7 +99,7 @@ jobs: freespace: true noros-dev: - needs: [check-changes, ros-python] + needs: [check-changes, python] if: | ${{ always() && !cancelled() && From 10c177509d56a83389d9cf68dc0ddfa29907a7c2 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 13:19:02 +0300 Subject: [PATCH 06/30] tests run on ros and noros --- .github/workflows/docker.yml | 18 ++++++++++++++++-- .github/workflows/tests.yml | 6 +++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fe425022d0..c02b4725e2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -128,7 +128,7 @@ jobs: branch-tag: ${{ needs.check-changes.outputs.branch-tag }} target: dev - run-tests: + run-ros-tests: needs: [check-changes, dev] if: | ${{ @@ -140,4 +140,18 @@ jobs: }} uses: ./.github/workflows/tests.yml with: - branch-tag: ${{ needs.dev.result != 'success' && 'dev' || needs.check-changes.outputs.branch-tag }} + dev-image: dev:${{ needs.dev.result != 'success' && 'dev' || needs.check-changes.outputs.branch-tag }} + + run-noros-tests: + needs: [check-changes, noros-dev] + if: | + ${{ + always() && !cancelled() && + needs.check-changes.result == 'success' && + ((needs.noros-dev.result == 'success') || + (needs.noros-dev.result == 'skipped' && + needs.check-changes.outputs.tests == 'true')) + }} + uses: ./.github/workflows/tests.yml + with: + dev-image: ${{ needs.noros-dev.result != 'success' && 'dev' || needs.check-changes.outputs.branch-tag }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7efc7bad01..44e40216b2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,10 +3,10 @@ name: tests on: workflow_call: inputs: - branch-tag: + dev-image: required: true type: string - default: "latest" + default: "dev:dev" permissions: contents: read @@ -17,7 +17,7 @@ jobs: runs-on: dimos-runner-ubuntu-2204 container: - image: ghcr.io/dimensionalos/dev:${{ inputs.branch-tag }} + image: ghcr.io/dimensionalos/${{ inputs.dev-image }} steps: - uses: actions/checkout@v4 From 8c69360a3142db76261f07c4cb39f900981de5c4 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 15:17:32 +0300 Subject: [PATCH 07/30] updating docker files and workflows for new builds --- .github/workflows/_docker-build-template.yml | 49 ++----- .github/workflows/docker.yml | 122 +++++++++--------- .github/workflows/tests.yml | 14 +- docker/dev/Dockerfile | 4 +- docker/{base-ros-python => python}/Dockerfile | 5 +- docker/{base-ros => ros}/Dockerfile | 4 +- docker/{base-ros => ros}/install-nix.sh | 0 7 files changed, 95 insertions(+), 103 deletions(-) rename docker/{base-ros-python => python}/Dockerfile (74%) rename docker/{base-ros => ros}/Dockerfile (98%) rename docker/{base-ros => ros}/install-nix.sh (100%) diff --git a/.github/workflows/_docker-build-template.yml b/.github/workflows/_docker-build-template.yml index 11861e8a9b..abe2bdfd52 100644 --- a/.github/workflows/_docker-build-template.yml +++ b/.github/workflows/_docker-build-template.yml @@ -2,10 +2,12 @@ name: docker-build-template on: workflow_call: inputs: - branch-tag: { type: string, required: true } - target: { type: string, required: true } + from-image: { type: string, required: true } + to-image: { type: string, required: true } + dockerfile: { type: string, required: true } freespace: { type: boolean, default: false } context: { type: string, default: '.' } + should-run: { type: boolean, default: false } # you can run this locally as well via # ./bin/dockerbuild [image-name] @@ -17,6 +19,10 @@ jobs: packages: write steps: + - name: exit early + if: ${{ !inputs.should-run }} + run: | + exit 0 - name: free up disk space # takes a bit of time, so disabled by default # explicitly enable this for large builds @@ -37,34 +43,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Check base image tag - id: tagcheck - env: - BRANCH_TAG: ${{ inputs.branch-tag }} - DOCKERFILE_PATH: docker/${{ inputs.target }}/Dockerfile - DOCKER_CLI_EXPERIMENTAL: enabled # required for `docker buildx imagetools` - run: | - BASE_REPO=$(grep -Eo '^FROM[[:space:]]+ghcr\.io/dimensionalos/[a-zA-Z0-9._-]+' "$DOCKERFILE_PATH" | head -n1 | awk -F/ '{print $NF}') - - if [[ -z "$BASE_REPO" ]]; then - echo "Could not determine base repo from $DOCKERFILE_PATH, reverting to 'dev'" - FROM_TAG="dev" - else - IMAGE="ghcr.io/dimensionalos/${BASE_REPO}:${BRANCH_TAG}" - echo "Checking if $IMAGE exists…" - if docker buildx imagetools inspect "$IMAGE" > /dev/null 2>&1; then - echo "Found $IMAGE" - FROM_TAG="$BRANCH_TAG" - else - echo "Tag '$BRANCH_TAG' not found for $BASE_REPO; falling back to 'dev'" - FROM_TAG="dev" - fi - fi - - echo "from_tag=$FROM_TAG" >> "$GITHUB_OUTPUT" - - - # required for github cache of docker layers - uses: crazy-max/ghaction-github-runtime@v3 @@ -79,9 +57,8 @@ jobs: with: push: true context: ${{ inputs.context }} - file: docker/${{ inputs.target }}/Dockerfile - tags: ghcr.io/dimensionalos/${{ inputs.target }}:${{ inputs.branch-tag }} - cache-from: type=gha,scope=${{ inputs.target }} - cache-to: type=gha,mode=max,scope=${{ inputs.target }} - build-args: | - FROM_TAG=${{ steps.tagcheck.outputs.from_tag }} + file: docker/${{ inputs.dockerfile }}/Dockerfile + tags: ${{ inputs.to-image }} + cache-from: type=gha,scope=${{ inputs.dockerfile }} + cache-to: type=gha,mode=max,scope=${{ inputs.dockerfile }} + build-args: FROM_IMAGE=${{ inputs.from-image }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c02b4725e2..0efb6a3229 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -44,7 +44,7 @@ jobs: id: set-tag run: | case "${GITHUB_REF_NAME}" in - master) branch_tag="latest" ;; + main) branch_tag="latest" ;; dev) branch_tag="dev" ;; *) branch_tag=$(echo "${GITHUB_REF_NAME}" \ @@ -61,8 +61,9 @@ jobs: if: needs.check-changes.outputs.ros == 'true' uses: ./.github/workflows/_docker-build-template.yml with: - branch-tag: ${{ needs.check-changes.outputs.branch-tag }} - target: base-ros + from-image: ubuntu:22.04 + to-image: ghcr.io/dimensionalos/ros:${{ needs.check-changes.outputs.branch-tag }} + dockerfile: ros # just a debugger inspect-needs: @@ -75,18 +76,19 @@ jobs: ros-python: needs: [check-changes, ros] - if: | - ${{ - always() && !cancelled() && - needs.check-changes.result == 'success' && - ((needs.ros.result == 'success') || - (needs.ros.result == 'skipped' && - needs.check-changes.outputs.python == 'true')) - }} + if: always() uses: ./.github/workflows/_docker-build-template.yml with: - branch-tag: ${{ needs.check-changes.outputs.branch-tag }} - target: base-ros-python + should-run: ${{ + !cancelled() && + needs.check-changes.outputs.python == 'true' && + needs.check-changes.result != 'error' && + needs.ros.result != 'error' + }} + + from-image: ghcr.io/dimensionalos/ros:${{ needs.ros.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} + to-image: ghcr.io/dimensionalos/ros-python:${{ needs.check-changes.outputs.branch-tag }} + dockerfile: python freespace: true python: @@ -94,64 +96,66 @@ jobs: if: needs.check-changes.outputs.python == 'true' uses: ./.github/workflows/_docker-build-template.yml with: - branch-tag: ${{ needs.check-changes.outputs.branch-tag }} - target: base-python + should-run: true freespace: true + dockerfile: python + from-image: ubuntu:22.04 + to-image: ghcr.io/dimensionalos/python:${{ needs.check-changes.outputs.branch-tag }} - noros-dev: + dev: needs: [check-changes, python] - if: | - ${{ - always() && !cancelled() && - needs.check-changes.result == 'success' && - ((needs.python.result == 'success') || - (needs.python.result == 'skipped' && - needs.check-changes.outputs.dev == 'true')) - }} + if: always() + uses: ./.github/workflows/_docker-build-template.yml with: - branch-tag: ${{ needs.check-changes.outputs.branch-tag }} - target: noros-dev + should-run: ${{ !cancelled() && + needs.check-changes.result == 'success' && + ((needs.python.result == 'success') || + (needs.python.result == 'skipped' && + needs.check-changes.outputs.dev == 'true')) }} + from-image: ghcr.io/dimensionalos/python:${{ needs.python.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} + to-image: ghcr.io/dimensionalos/dev:${{ needs.check-changes.outputs.branch-tag }} + dockerfile: dev - dev: + ros-dev: needs: [check-changes, ros-python] - if: | - ${{ - always() && !cancelled() && - needs.check-changes.result == 'success' && - ((needs.ros-python.result == 'success') || - (needs.ros-python.result == 'skipped' && - needs.check-changes.outputs.dev == 'true')) - }} + if: always() uses: ./.github/workflows/_docker-build-template.yml with: - branch-tag: ${{ needs.check-changes.outputs.branch-tag }} - target: dev + should-run: ${{ !cancelled() && + needs.check-changes.result == 'success' && + ((needs.ros-python.result == 'success') || + (needs.ros-python.result == 'skipped' && + needs.check-changes.outputs.dev == 'true')) + }} + from-image: ghcr.io/dimensionalos/ros-python:${{ needs.ros-python.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} + to-image: ghcr.io/dimensionalos/ros-dev:${{ needs.check-changes.outputs.branch-tag }} + dockerfile: dev run-ros-tests: - needs: [check-changes, dev] - if: | - ${{ - always() && !cancelled() && - needs.check-changes.result == 'success' && - ((needs.dev.result == 'success') || - (needs.dev.result == 'skipped' && - needs.check-changes.outputs.tests == 'true')) - }} + needs: [check-changes, ros-dev] + if: always() uses: ./.github/workflows/tests.yml with: - dev-image: dev:${{ needs.dev.result != 'success' && 'dev' || needs.check-changes.outputs.branch-tag }} - - run-noros-tests: - needs: [check-changes, noros-dev] - if: | - ${{ - always() && !cancelled() && - needs.check-changes.result == 'success' && - ((needs.noros-dev.result == 'success') || - (needs.noros-dev.result == 'skipped' && - needs.check-changes.outputs.tests == 'true')) - }} + should-run: ${{ !cancelled() && + needs.check-changes.result == 'success' && + ((needs.ros-dev.result == 'success') || + (needs.ros-dev.result == 'skipped' && + needs.check-changes.outputs.tests == 'true')) + }} + cmd: "pytest -m ros" # enable tests depend on ros + dev-image: ros-dev:${{ needs.ros-dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} + + run-tests: + needs: [check-changes, dev] + if: always() uses: ./.github/workflows/tests.yml with: - dev-image: ${{ needs.noros-dev.result != 'success' && 'dev' || needs.check-changes.outputs.branch-tag }} + should-run: ${{ !cancelled() && + needs.check-changes.result == 'success' && + ((needs.dev.result == 'success') || + (needs.dev.result == 'skipped' && + needs.check-changes.outputs.tests == 'true')) + }} + cmd: "pytest" + dev-image: dev:${{ needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 44e40216b2..a9cdb78abf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,10 +3,17 @@ name: tests on: workflow_call: inputs: + should-run: + required: false + type: boolean + default: true dev-image: required: true type: string default: "dev:dev" + cmd: + required: true + type: string permissions: contents: read @@ -20,9 +27,14 @@ jobs: image: ghcr.io/dimensionalos/${{ inputs.dev-image }} steps: + - name: exit early + if: ${{ !inputs.should-run }} + run: | + exit 0 + - uses: actions/checkout@v4 - name: Run tests run: | git config --global --add safe.directory '*' - /entrypoint.sh bash -c "pytest" + /entrypoint.sh bash -c "${{ inputs.cmd }}" diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index a210fd4e15..195b87d117 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,5 +1,5 @@ -ARG FROM_TAG=latest -FROM ghcr.io/dimensionalos/base-ros-python:${FROM_TAG} +ARG FROM_TAG +FROM ${FROM_IMAGE} ARG GIT_COMMIT=unknown ARG GIT_BRANCH=unknown diff --git a/docker/base-ros-python/Dockerfile b/docker/python/Dockerfile similarity index 74% rename from docker/base-ros-python/Dockerfile rename to docker/python/Dockerfile index f86974b41a..d91d79c6a1 100644 --- a/docker/base-ros-python/Dockerfile +++ b/docker/python/Dockerfile @@ -1,6 +1,5 @@ -# trigger rebuild: 1 -ARG FROM_TAG=latest -FROM ghcr.io/dimensionalos/base-ros:${FROM_TAG} +ARG FROM_IMAGE +FROM ${FROM_IMAGE} RUN mkdir -p /app/dimos diff --git a/docker/base-ros/Dockerfile b/docker/ros/Dockerfile similarity index 98% rename from docker/base-ros/Dockerfile rename to docker/ros/Dockerfile index b1814ba54b..df6544f23b 100644 --- a/docker/base-ros/Dockerfile +++ b/docker/ros/Dockerfile @@ -1,5 +1,5 @@ -# trigger rebuild: 1 -FROM ubuntu:22.04 +ARG FROM_IMAGE=ubuntu:22.04 +FROM ${FROM_IMAGE} # Avoid prompts from apt ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/base-ros/install-nix.sh b/docker/ros/install-nix.sh similarity index 100% rename from docker/base-ros/install-nix.sh rename to docker/ros/install-nix.sh From e9826275b52d29ffcd5462908a4caf69c6eaf754 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 15:18:48 +0300 Subject: [PATCH 08/30] workflow bugfix --- .github/workflows/docker.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0efb6a3229..00416df132 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -80,7 +80,6 @@ jobs: uses: ./.github/workflows/_docker-build-template.yml with: should-run: ${{ - !cancelled() && needs.check-changes.outputs.python == 'true' && needs.check-changes.result != 'error' && needs.ros.result != 'error' @@ -108,7 +107,7 @@ jobs: uses: ./.github/workflows/_docker-build-template.yml with: - should-run: ${{ !cancelled() && + should-run: ${{ needs.check-changes.result == 'success' && ((needs.python.result == 'success') || (needs.python.result == 'skipped' && @@ -122,7 +121,7 @@ jobs: if: always() uses: ./.github/workflows/_docker-build-template.yml with: - should-run: ${{ !cancelled() && + should-run: ${{ needs.check-changes.result == 'success' && ((needs.ros-python.result == 'success') || (needs.ros-python.result == 'skipped' && @@ -137,7 +136,7 @@ jobs: if: always() uses: ./.github/workflows/tests.yml with: - should-run: ${{ !cancelled() && + should-run: ${{ needs.check-changes.result == 'success' && ((needs.ros-dev.result == 'success') || (needs.ros-dev.result == 'skipped' && @@ -151,7 +150,7 @@ jobs: if: always() uses: ./.github/workflows/tests.yml with: - should-run: ${{ !cancelled() && + should-run: ${{ needs.check-changes.result == 'success' && ((needs.dev.result == 'success') || (needs.dev.result == 'skipped' && From 229be9d77cb93bbfd4e5507c312cf207e37d6458 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 15:25:48 +0300 Subject: [PATCH 09/30] debug job is not neccessary --- .github/workflows/docker.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 00416df132..32df12d1f2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -65,14 +65,14 @@ jobs: to-image: ghcr.io/dimensionalos/ros:${{ needs.check-changes.outputs.branch-tag }} dockerfile: ros - # just a debugger - inspect-needs: - needs: [check-changes, ros] - runs-on: dimos-runner-ubuntu-2204 - if: always() - steps: - - run: | - echo '${{ toJSON(needs) }}' + # # just a debugger + # inspect-needs: + # needs: [check-changes, ros] + # runs-on: dimos-runner-ubuntu-2204 + # if: always() + # steps: + # - run: | + # echo '${{ toJSON(needs) }}' ros-python: needs: [check-changes, ros] From 5d6c6cd25527adb1d510aca29173c590d8d09b49 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 15:29:39 +0300 Subject: [PATCH 10/30] python image needs system deps --- docker/python/Dockerfile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile index d91d79c6a1..86cedbb9f3 100644 --- a/docker/python/Dockerfile +++ b/docker/python/Dockerfile @@ -1,6 +1,36 @@ ARG FROM_IMAGE FROM ${FROM_IMAGE} +# Install basic requirements +RUN apt-get update +RUN apt-get install -y \ + curl \ + gnupg2 \ + lsb-release \ + python3-pip \ + clang \ + portaudio19-dev \ + git \ + mesa-utils \ + libgl1-mesa-glx \ + libgl1-mesa-dri \ + software-properties-common \ + libxcb1-dev \ + libxcb-keysyms1-dev \ + libxcb-util0-dev \ + libxcb-icccm4-dev \ + libxcb-image0-dev \ + libxcb-randr0-dev \ + libxcb-shape0-dev \ + libxcb-xinerama0-dev \ + libxcb-xkb-dev \ + libxkbcommon-x11-dev \ + qtbase5-dev \ + qtchooser \ + qt5-qmake \ + qtbase5-dev-tools \ + supervisor + RUN mkdir -p /app/dimos COPY requirements.txt /app/ From 52b9aeebf754387207d32b178e331ce9f46d5fa7 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 16:33:34 +0300 Subject: [PATCH 11/30] dockerfile fix --- docker/dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 195b87d117..2baed56981 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM_TAG +ARG FROM_IMAGE FROM ${FROM_IMAGE} ARG GIT_COMMIT=unknown From 1b384916af6c004cd60f7b0fe1bf6520a493938b Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 18:40:50 +0300 Subject: [PATCH 12/30] added ros polyfill --- dimos/types/ros_polyfill.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 dimos/types/ros_polyfill.py diff --git a/dimos/types/ros_polyfill.py b/dimos/types/ros_polyfill.py new file mode 100644 index 0000000000..416ff3b522 --- /dev/null +++ b/dimos/types/ros_polyfill.py @@ -0,0 +1,26 @@ +# Copyright 2025 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + from geometry_msgs.msg import Vector3 +except ImportError: + + class Vector3: + def __init__(self, x: float = 0.0, y: float = 0.0, z: float = 0.0): + self.x = float(x) + self.y = float(y) + self.z = float(z) + + def __repr__(self) -> str: + return f"Vector3(x={self.x}, y={self.y}, z={self.z})" From 31e711fb95f35c5ad391e23ec0b76b908ab69e9a Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 19:01:43 +0300 Subject: [PATCH 13/30] ros tests fix, debug job re-introduced --- .github/workflows/_docker-build-template.yml | 6 +++--- .github/workflows/docker.yml | 2 +- docs/ci.md | 18 +++++++++--------- pyproject.toml | 3 ++- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/_docker-build-template.yml b/.github/workflows/_docker-build-template.yml index abe2bdfd52..d45e88afbc 100644 --- a/.github/workflows/_docker-build-template.yml +++ b/.github/workflows/_docker-build-template.yml @@ -3,11 +3,11 @@ on: workflow_call: inputs: from-image: { type: string, required: true } - to-image: { type: string, required: true } - dockerfile: { type: string, required: true } + to-image: { type: string, required: true } + dockerfile: { type: string, required: true } freespace: { type: boolean, default: false } - context: { type: string, default: '.' } should-run: { type: boolean, default: false } + context: { type: string, default: '.' } # you can run this locally as well via # ./bin/dockerbuild [image-name] diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 32df12d1f2..f89022c424 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -142,7 +142,7 @@ jobs: (needs.ros-dev.result == 'skipped' && needs.check-changes.outputs.tests == 'true')) }} - cmd: "pytest -m ros" # enable tests depend on ros + cmd: "pytest && pytest -m ros" # run tests that depend on ros as well dev-image: ros-dev:${{ needs.ros-dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} run-tests: diff --git a/docs/ci.md b/docs/ci.md index ee4182e8ea..5cb05393a6 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -52,15 +52,15 @@ pytest # run tests ``` - ┌───┐ - │ros│ - └┬──┘ - ┌▽─────┐ - │python│ - └┬─────┘ - ┌▽──┐ - │dev│ - └───┘ + ┌───┐┌────────┐ + │ros││python │ + └┬──┘└───────┬┘ + ┌▽─────────┐┌▽──┐ + │ros-python││dev│ + └┬─────────┘└───┘ + ┌▽──────┐ + │ros-dev│ + └───────┘ ``` > **Note**: The diagram shows only currently active images; the system is extensible—new combinations are possible, builds can be run per branch and as parallel as possible diff --git a/pyproject.toml b/pyproject.toml index bbee605a7f..5bab3b2631 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ markers = [ "benchmark: benchmark, executes something multiple times, calculates avg, prints to console", "exclude: arbitrary exclusion from CI and default test exec", "tool: dev tooling", - "needsdata: needs test data to be downloaded"] + "needsdata: needs test data to be downloaded", + "ros: depend on ros"] addopts = "-v -ra --color=yes -m 'not vis and not benchmark and not exclude and not tool and not needsdata'" From b00cd38692135f83104d166404cce3f164275f50 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 19:04:25 +0300 Subject: [PATCH 14/30] reintroduced the debug step --- .github/workflows/docker.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f89022c424..0de7cc6abe 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -56,6 +56,15 @@ jobs: echo "branch tag determined: ${branch_tag}" echo branch_tag="${branch_tag}" >> "$GITHUB_OUTPUT" + # just a debugger + inspect-needs: + needs: [check-changes, ros] + runs-on: dimos-runner-ubuntu-2204 + if: always() + steps: + - run: | + echo '${{ toJSON(needs) }}' + ros: needs: [check-changes] if: needs.check-changes.outputs.ros == 'true' From b79bfea74a752c7259595cf123bc83212fc52eee Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 19:11:32 +0300 Subject: [PATCH 15/30] devcontainer should use ros-dev image --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 860fcd87f2..29ef16fb81 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "dimos-dev", - "image": "ghcr.io/dimensionalos/dev:dev", + "image": "ghcr.io/dimensionalos/ros-dev:dev", "customizations": { "vscode": { "extensions": [ From 711d6a492acb9d359220841863bfa25563666a44 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 19:16:21 +0300 Subject: [PATCH 16/30] ros test polyfil --- dimos/types/test_ros_polyfil.py | 22 ++++++++++++++++++++++ docs/ci.md | 5 ++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 dimos/types/test_ros_polyfil.py diff --git a/dimos/types/test_ros_polyfil.py b/dimos/types/test_ros_polyfil.py new file mode 100644 index 0000000000..56faecc0b5 --- /dev/null +++ b/dimos/types/test_ros_polyfil.py @@ -0,0 +1,22 @@ +# Copyright 2025 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.mark.ros +def test_ros_polyfil(): + # just so we have some ros-specific tests, otherwise + # pytest -m ros exits with code 5 + return True diff --git a/docs/ci.md b/docs/ci.md index 5cb05393a6..1c638235e4 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -52,7 +52,10 @@ pytest # run tests ``` - ┌───┐┌────────┐ + ┌──────┐ + │ubuntu│ + └┬────┬┘ + ┌▽──┐┌▽───────┐ │ros││python │ └┬──┘└───────┬┘ ┌▽─────────┐┌▽──┐ From f1458653e975416aa3dd403d5fb3a0f7754bbd27 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 19:30:39 +0300 Subject: [PATCH 17/30] nav_msgs polyfil --- dimos/types/costmap.py | 2 +- dimos/types/ros_polyfill.py | 59 +++++++++++++++++++++++++++++++++++++ docs/ci.md | 6 ++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/dimos/types/costmap.py b/dimos/types/costmap.py index b349af7fd5..1107abf8bf 100644 --- a/dimos/types/costmap.py +++ b/dimos/types/costmap.py @@ -17,7 +17,7 @@ import numpy as np from typing import Optional from scipy import ndimage -from nav_msgs.msg import OccupancyGrid +from dimos.types.ros_polyfill import OccupancyGrid from dimos.types.vector import Vector, VectorLike, x, y, to_vector import open3d as o3d diff --git a/dimos/types/ros_polyfill.py b/dimos/types/ros_polyfill.py index 416ff3b522..47eac7c5d1 100644 --- a/dimos/types/ros_polyfill.py +++ b/dimos/types/ros_polyfill.py @@ -24,3 +24,62 @@ def __init__(self, x: float = 0.0, y: float = 0.0, z: float = 0.0): def __repr__(self) -> str: return f"Vector3(x={self.x}, y={self.y}, z={self.z})" + + +try: + from nav_msgs.msg import OccupancyGrid + from geometry_msgs.msg import Pose, Point, Quaternion + from std_msgs.msg import Header +except ImportError: + + class Header: + def __init__(self): + self.stamp = None + self.frame_id = "" + + class Point: + def __init__(self, x: float = 0.0, y: float = 0.0, z: float = 0.0): + self.x = float(x) + self.y = float(y) + self.z = float(z) + + def __repr__(self) -> str: + return f"Point(x={self.x}, y={self.y}, z={self.z})" + + class Quaternion: + def __init__(self, x: float = 0.0, y: float = 0.0, z: float = 0.0, w: float = 1.0): + self.x = float(x) + self.y = float(y) + self.z = float(z) + self.w = float(w) + + def __repr__(self) -> str: + return f"Quaternion(x={self.x}, y={self.y}, z={self.z}, w={self.w})" + + class Pose: + def __init__(self): + self.position = Point() + self.orientation = Quaternion() + + def __repr__(self) -> str: + return f"Pose(position={self.position}, orientation={self.orientation})" + + class MapMetaData: + def __init__(self): + self.map_load_time = None + self.resolution = 0.05 + self.width = 0 + self.height = 0 + self.origin = Pose() + + def __repr__(self) -> str: + return f"MapMetaData(resolution={self.resolution}, width={self.width}, height={self.height}, origin={self.origin})" + + class OccupancyGrid: + def __init__(self): + self.header = Header() + self.info = MapMetaData() + self.data = [] + + def __repr__(self) -> str: + return f"OccupancyGrid(info={self.info}, data_length={len(self.data)})" diff --git a/docs/ci.md b/docs/ci.md index 1c638235e4..a041ab08cc 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -66,6 +66,12 @@ pytest # run tests └───────┘ ``` +* ghcr.io/dimensionalos/ros:dev +* ghcr.io/dimensionalos/python:dev +* ghcr.io/dimensionalos/ros-python:dev +* ghcr.io/dimensionalos/ros-dev:dev +* ghcr.io/dimensionalos/dev:dev + > **Note**: The diagram shows only currently active images; the system is extensible—new combinations are possible, builds can be run per branch and as parallel as possible From 903280c2e212102454d2992fdbdd395da2f2b0df Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 19:51:39 +0300 Subject: [PATCH 18/30] observable topic polyfill --- dimos/robot/ros_observable_topic.py | 8 ++++---- dimos/robot/test_ros_observable_topic.py | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dimos/robot/ros_observable_topic.py b/dimos/robot/ros_observable_topic.py index 697ddff398..d070f4ed70 100644 --- a/dimos/robot/ros_observable_topic.py +++ b/dimos/robot/ros_observable_topic.py @@ -20,11 +20,11 @@ from reactivex.scheduler import ThreadPoolScheduler from rxpy_backpressure import BackPressure -from nav_msgs import msg from dimos.utils.logging_config import setup_logger from dimos.utils.threadpool import get_scheduler from dimos.types.costmap import Costmap from dimos.types.vector import Vector +from dimos.types.ros_polyfill import OccupancyGrid, Odometry from typing import Union, Callable, Any @@ -38,7 +38,7 @@ __all__ = ["ROSObservableTopicAbility", "QOS"] ConversionType = Costmap -TopicType = Union[ConversionType, msg.OccupancyGrid, msg.Odometry] +TopicType = Union[ConversionType, OccupancyGrid, Odometry] class QOS(enum.Enum): @@ -91,10 +91,10 @@ def _maybe_conversion(self, msg_type: TopicType, callback) -> Callable[[TopicTyp def _sub_msg_type(self, msg_type): if msg_type == Costmap: - return msg.OccupancyGrid + return OccupancyGrid if msg_type == Vector: - return msg.Odometry + return Odometry return msg_type diff --git a/dimos/robot/test_ros_observable_topic.py b/dimos/robot/test_ros_observable_topic.py index 4df9dbb0de..9676cee7f7 100644 --- a/dimos/robot/test_ros_observable_topic.py +++ b/dimos/robot/test_ros_observable_topic.py @@ -15,14 +15,18 @@ import threading import time -from nav_msgs import msg import pytest from dimos.robot.ros_observable_topic import ROSObservableTopicAbility from dimos.utils.logging_config import setup_logger from dimos.types.vector import Vector +from dimos.types.ros_polyfill import Odometry import asyncio +class msg: + Odometry = Odometry + + class MockROSNode: def __init__(self): self.logger = setup_logger("ROS") From b5589aad75adcde4d2b63816f1b2abd61359bbe5 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 19:53:46 +0300 Subject: [PATCH 19/30] ros observable topic is tested only on the ros image --- dimos/robot/ros_observable_topic.py | 8 +++---- dimos/robot/test_ros_observable_topic.py | 28 ++++++++++++++---------- dimos/types/ros_polyfill.py | 22 +++++++++++++++++-- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/dimos/robot/ros_observable_topic.py b/dimos/robot/ros_observable_topic.py index d070f4ed70..697ddff398 100644 --- a/dimos/robot/ros_observable_topic.py +++ b/dimos/robot/ros_observable_topic.py @@ -20,11 +20,11 @@ from reactivex.scheduler import ThreadPoolScheduler from rxpy_backpressure import BackPressure +from nav_msgs import msg from dimos.utils.logging_config import setup_logger from dimos.utils.threadpool import get_scheduler from dimos.types.costmap import Costmap from dimos.types.vector import Vector -from dimos.types.ros_polyfill import OccupancyGrid, Odometry from typing import Union, Callable, Any @@ -38,7 +38,7 @@ __all__ = ["ROSObservableTopicAbility", "QOS"] ConversionType = Costmap -TopicType = Union[ConversionType, OccupancyGrid, Odometry] +TopicType = Union[ConversionType, msg.OccupancyGrid, msg.Odometry] class QOS(enum.Enum): @@ -91,10 +91,10 @@ def _maybe_conversion(self, msg_type: TopicType, callback) -> Callable[[TopicTyp def _sub_msg_type(self, msg_type): if msg_type == Costmap: - return OccupancyGrid + return msg.OccupancyGrid if msg_type == Vector: - return Odometry + return msg.Odometry return msg_type diff --git a/dimos/robot/test_ros_observable_topic.py b/dimos/robot/test_ros_observable_topic.py index 9676cee7f7..ebc5a3a1d6 100644 --- a/dimos/robot/test_ros_observable_topic.py +++ b/dimos/robot/test_ros_observable_topic.py @@ -19,14 +19,9 @@ from dimos.robot.ros_observable_topic import ROSObservableTopicAbility from dimos.utils.logging_config import setup_logger from dimos.types.vector import Vector -from dimos.types.ros_polyfill import Odometry import asyncio -class msg: - Odometry = Odometry - - class MockROSNode: def __init__(self): self.logger = setup_logger("ROS") @@ -86,7 +81,10 @@ def __init__(self): # 3. that the system unsubscribes from ROS when observers are disposed # 4. that the system replays the last message to new observers, # before the new ROS sub starts producing +@pytest.mark.ros def test_parallel_and_cleanup(): + from nav_msgs import msg + robot = MockRobot() received_messages = [] @@ -156,7 +154,10 @@ def test_parallel_and_cleanup(): # ROS thread ─► ReplaySubject─► observe_on(pool) ─► backpressure.latest ─► sub1 (fast) # ├──► observe_on(pool) ─► backpressure.latest ─► sub2 (slow) # └──► observe_on(pool) ─► backpressure.latest ─► sub3 (slower) +@pytest.mark.ros def test_parallel_and_hog(): + from nav_msgs import msg + robot = MockRobot() obs1 = robot.topic("/odom", msg.Odometry) @@ -195,7 +196,10 @@ def test_parallel_and_hog(): @pytest.mark.asyncio +@pytest.mark.ros async def test_topic_latest_async(): + from nav_msgs import msg + robot = MockRobot() odom = await robot.topic_latest_async("/odom", msg.Odometry) @@ -207,6 +211,7 @@ async def test_topic_latest_async(): assert robot._node.subs == {} +@pytest.mark.ros def test_topic_auto_conversion(): robot = MockRobot() odom = robot.topic("/vector", Vector).subscribe(lambda x: print(x)) @@ -214,7 +219,10 @@ def test_topic_auto_conversion(): odom.dispose() +@pytest.mark.ros def test_topic_latest_sync(): + from nav_msgs import msg + robot = MockRobot() odom = robot.topic_latest("/odom", msg.Odometry) @@ -226,7 +234,10 @@ def test_topic_latest_sync(): assert robot._node.subs == {} +@pytest.mark.ros def test_topic_latest_sync_benchmark(): + from nav_msgs import msg + robot = MockRobot() odom = robot.topic_latest("/odom", msg.Odometry) @@ -246,10 +257,3 @@ def test_topic_latest_sync_benchmark(): odom.dispose() time.sleep(0.1) assert robot._node.subs == {} - - -if __name__ == "__main__": - test_parallel_and_cleanup() - test_parallel_and_hog() - test_topic_latest_sync() - asyncio.run(test_topic_latest_async()) diff --git a/dimos/types/ros_polyfill.py b/dimos/types/ros_polyfill.py index 47eac7c5d1..b5c2bc1d64 100644 --- a/dimos/types/ros_polyfill.py +++ b/dimos/types/ros_polyfill.py @@ -27,8 +27,8 @@ def __repr__(self) -> str: try: - from nav_msgs.msg import OccupancyGrid - from geometry_msgs.msg import Pose, Point, Quaternion + from nav_msgs.msg import OccupancyGrid, Odometry + from geometry_msgs.msg import Pose, Point, Quaternion, Twist from std_msgs.msg import Header except ImportError: @@ -75,6 +75,14 @@ def __init__(self): def __repr__(self) -> str: return f"MapMetaData(resolution={self.resolution}, width={self.width}, height={self.height}, origin={self.origin})" + class Twist: + def __init__(self): + self.linear = Vector3() + self.angular = Vector3() + + def __repr__(self) -> str: + return f"Twist(linear={self.linear}, angular={self.angular})" + class OccupancyGrid: def __init__(self): self.header = Header() @@ -83,3 +91,13 @@ def __init__(self): def __repr__(self) -> str: return f"OccupancyGrid(info={self.info}, data_length={len(self.data)})" + + class Odometry: + def __init__(self): + self.header = Header() + self.child_frame_id = "" + self.pose = Pose() + self.twist = Twist() + + def __repr__(self) -> str: + return f"Odometry(pose={self.pose}, twist={self.twist})" From 77b21feaa06749f07065e616502f848c7c817ba5 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 20:01:43 +0300 Subject: [PATCH 20/30] removed stub polyfill test - we have other ros tests now --- dimos/types/test_ros_polyfil.py | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 dimos/types/test_ros_polyfil.py diff --git a/dimos/types/test_ros_polyfil.py b/dimos/types/test_ros_polyfil.py deleted file mode 100644 index 56faecc0b5..0000000000 --- a/dimos/types/test_ros_polyfil.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2025 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - - -@pytest.mark.ros -def test_ros_polyfil(): - # just so we have some ros-specific tests, otherwise - # pytest -m ros exits with code 5 - return True From b14b5e081ceb59af110ac2f93364b633a3de2e5d Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 20:36:59 +0300 Subject: [PATCH 21/30] entrypoint fix --- docker/dev/entrypoint.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/dev/entrypoint.sh b/docker/dev/entrypoint.sh index fb78050376..d48bea16e3 100644 --- a/docker/dev/entrypoint.sh +++ b/docker/dev/entrypoint.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash -source /opt/ros/${ROS_DISTRO}/setup.bash -#source /ros2_ws/install/setup.bash +if [ -d "/opt/ros/${ROS_DISTRO}" ]; then + source /opt/ros/${ROS_DISTRO}/setup.bash +else + echo "ROS is not available in this env" +fi exec "$@" From f6c8fd445f8177f4a98878d717c1269b9e0c9897 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 20:48:15 +0300 Subject: [PATCH 22/30] observable topic fix --- dimos/robot/test_ros_observable_topic.py | 40 +++++++++++------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/dimos/robot/test_ros_observable_topic.py b/dimos/robot/test_ros_observable_topic.py index ebc5a3a1d6..2885c8649b 100644 --- a/dimos/robot/test_ros_observable_topic.py +++ b/dimos/robot/test_ros_observable_topic.py @@ -16,7 +16,6 @@ import threading import time import pytest -from dimos.robot.ros_observable_topic import ROSObservableTopicAbility from dimos.utils.logging_config import setup_logger from dimos.types.vector import Vector import asyncio @@ -67,11 +66,18 @@ def destroy_subscription(self, subscription): self.logger.info(f"Unknown subscription: {subscription}") -class MockRobot(ROSObservableTopicAbility): - def __init__(self): - self.logger = setup_logger("ROBOT") - # Initialize the mock ROS node - self._node = MockROSNode() +# we are doing this in order to avoid importing ROS dependencies +@pytest.fixture +def robot(): + from dimos.robot.ros_observable_topic import ROSObservableTopicAbility + + class MockRobot(ROSObservableTopicAbility): + def __init__(self): + self.logger = setup_logger("ROBOT") + # Initialize the mock ROS node + self._node = MockROSNode() + + return MockRobot() # This test verifies a bunch of basics: @@ -82,10 +88,9 @@ def __init__(self): # 4. that the system replays the last message to new observers, # before the new ROS sub starts producing @pytest.mark.ros -def test_parallel_and_cleanup(): +def test_parallel_and_cleanup(robot): from nav_msgs import msg - robot = MockRobot() received_messages = [] obs1 = robot.topic("/odom", msg.Odometry) @@ -155,11 +160,9 @@ def test_parallel_and_cleanup(): # ├──► observe_on(pool) ─► backpressure.latest ─► sub2 (slow) # └──► observe_on(pool) ─► backpressure.latest ─► sub3 (slower) @pytest.mark.ros -def test_parallel_and_hog(): +def test_parallel_and_hog(robot): from nav_msgs import msg - robot = MockRobot() - obs1 = robot.topic("/odom", msg.Odometry) obs2 = robot.topic("/odom", msg.Odometry) @@ -197,11 +200,9 @@ def test_parallel_and_hog(): @pytest.mark.asyncio @pytest.mark.ros -async def test_topic_latest_async(): +async def test_topic_latest_async(robot): from nav_msgs import msg - robot = MockRobot() - odom = await robot.topic_latest_async("/odom", msg.Odometry) assert odom() == 1 await asyncio.sleep(0.45) @@ -212,19 +213,16 @@ async def test_topic_latest_async(): @pytest.mark.ros -def test_topic_auto_conversion(): - robot = MockRobot() +def test_topic_auto_conversion(robot): odom = robot.topic("/vector", Vector).subscribe(lambda x: print(x)) time.sleep(0.5) odom.dispose() @pytest.mark.ros -def test_topic_latest_sync(): +def test_topic_latest_sync(robot): from nav_msgs import msg - robot = MockRobot() - odom = robot.topic_latest("/odom", msg.Odometry) assert odom() == 1 time.sleep(0.45) @@ -235,11 +233,9 @@ def test_topic_latest_sync(): @pytest.mark.ros -def test_topic_latest_sync_benchmark(): +def test_topic_latest_sync_benchmark(robot): from nav_msgs import msg - robot = MockRobot() - odom = robot.topic_latest("/odom", msg.Odometry) start_time = time.time() From 000064d1d448cd2f1f24a89421cd4c4a97c7e7d6 Mon Sep 17 00:00:00 2001 From: lesh Date: Thu, 5 Jun 2025 21:02:14 +0300 Subject: [PATCH 23/30] triggering CI --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0de7cc6abe..72aba8a359 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,3 +1,4 @@ +# trigger build: 1 name: docker on: workflow_call From 99e43939df7c61f440a9ec964a2a98d5ab1fe24a Mon Sep 17 00:00:00 2001 From: lesh Date: Fri, 6 Jun 2025 13:12:21 +0300 Subject: [PATCH 24/30] odometry test fix --- dimos/robot/unitree_webrtc/type/test_odometry.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dimos/robot/unitree_webrtc/type/test_odometry.py b/dimos/robot/unitree_webrtc/type/test_odometry.py index 74fa512177..2e3ee9758e 100644 --- a/dimos/robot/unitree_webrtc/type/test_odometry.py +++ b/dimos/robot/unitree_webrtc/type/test_odometry.py @@ -14,19 +14,16 @@ from __future__ import annotations -from functools import reduce -import math from operator import sub, add import os import threading -from typing import Iterable, Optional +from typing import Optional import reactivex.operators as ops import pytest from dotenv import load_dotenv -from dimos.robot.unitree_webrtc.type.odometry import Odometry, RawOdometryMessage -from dimos.robot.unitree_webrtc.unitree_go2 import UnitreeGo2 +from dimos.robot.unitree_webrtc.type.odometry import Odometry from dimos.utils.testing import SensorReplay, SensorStorage _EXPECTED_TOTAL_RAD = -4.05212 @@ -91,6 +88,8 @@ def test_total_rotation_travel_rxpy() -> None: # data collection tool @pytest.mark.tool def test_store_odometry_stream() -> None: + from dimos.robot.unitree_webrtc.unitree_go2 import UnitreeGo2 + load_dotenv() robot = UnitreeGo2(ip=os.getenv("ROBOT_IP"), mode="ai") From c73a09ee57ae4d72ed905077115d1da295ca303e Mon Sep 17 00:00:00 2001 From: lesh Date: Fri, 6 Jun 2025 13:15:35 +0300 Subject: [PATCH 25/30] local planner decoupled from ros --- dimos/robot/local_planner/local_planner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dimos/robot/local_planner/local_planner.py b/dimos/robot/local_planner/local_planner.py index 4bd0b4a790..2c559afae6 100644 --- a/dimos/robot/local_planner/local_planner.py +++ b/dimos/robot/local_planner/local_planner.py @@ -30,7 +30,7 @@ from dimos.types.vector import VectorLike, Vector, to_tuple from dimos.types.path import Path -from nav_msgs.msg import OccupancyGrid +from dimos.types.costmap import Costmap logger = setup_logger("dimos.robot.unitree.local_planner", level=logging.DEBUG) @@ -45,7 +45,7 @@ class BaseLocalPlanner(ABC): def __init__( self, - get_costmap: Callable[[], Optional[OccupancyGrid]], + get_costmap: Callable[[], Optional[Costmap]], transform: object, move_vel_control: Callable[[float, float, float], None], safety_threshold: float = 0.5, From 50e979823c4cdcf4522ba080eab23c25ecf5a5aa Mon Sep 17 00:00:00 2001 From: lesh Date: Fri, 6 Jun 2025 14:03:07 +0300 Subject: [PATCH 26/30] kicked out fixture from ros_observable_topic tests --- dimos/robot/test_ros_observable_topic.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/dimos/robot/test_ros_observable_topic.py b/dimos/robot/test_ros_observable_topic.py index 2885c8649b..0f14f9f5bf 100644 --- a/dimos/robot/test_ros_observable_topic.py +++ b/dimos/robot/test_ros_observable_topic.py @@ -66,9 +66,8 @@ def destroy_subscription(self, subscription): self.logger.info(f"Unknown subscription: {subscription}") -# we are doing this in order to avoid importing ROS dependencies -@pytest.fixture -def robot(): +# we are doing this in order to avoid importing ROS dependencies if ros tests aren't runnin +def init_robot(): from dimos.robot.ros_observable_topic import ROSObservableTopicAbility class MockRobot(ROSObservableTopicAbility): @@ -88,7 +87,8 @@ def __init__(self): # 4. that the system replays the last message to new observers, # before the new ROS sub starts producing @pytest.mark.ros -def test_parallel_and_cleanup(robot): +def test_parallel_and_cleanup(): + robot = init_robot() from nav_msgs import msg received_messages = [] @@ -160,7 +160,8 @@ def test_parallel_and_cleanup(robot): # ├──► observe_on(pool) ─► backpressure.latest ─► sub2 (slow) # └──► observe_on(pool) ─► backpressure.latest ─► sub3 (slower) @pytest.mark.ros -def test_parallel_and_hog(robot): +def test_parallel_and_hog(): + robot = init_robot() from nav_msgs import msg obs1 = robot.topic("/odom", msg.Odometry) @@ -200,7 +201,8 @@ def test_parallel_and_hog(robot): @pytest.mark.asyncio @pytest.mark.ros -async def test_topic_latest_async(robot): +async def test_topic_latest_async(): + robot = init_robot() from nav_msgs import msg odom = await robot.topic_latest_async("/odom", msg.Odometry) @@ -213,14 +215,16 @@ async def test_topic_latest_async(robot): @pytest.mark.ros -def test_topic_auto_conversion(robot): +def test_topic_auto_conversion(): + robot = init_robot() odom = robot.topic("/vector", Vector).subscribe(lambda x: print(x)) time.sleep(0.5) odom.dispose() @pytest.mark.ros -def test_topic_latest_sync(robot): +def test_topic_latest_sync(): + robot = init_robot() from nav_msgs import msg odom = robot.topic_latest("/odom", msg.Odometry) @@ -233,7 +237,8 @@ def test_topic_latest_sync(robot): @pytest.mark.ros -def test_topic_latest_sync_benchmark(robot): +def test_topic_latest_sync_benchmark(): + robot = init_robot() from nav_msgs import msg odom = robot.topic_latest("/odom", msg.Odometry) From 7b565d325717ea7cc1fe4ce18a1879d45080aed9 Mon Sep 17 00:00:00 2001 From: lesh Date: Fri, 6 Jun 2025 14:24:29 +0300 Subject: [PATCH 27/30] sorted ros test exclusion --- dimos/robot/test_ros_observable_topic.py | 21 ++++++++------------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/dimos/robot/test_ros_observable_topic.py b/dimos/robot/test_ros_observable_topic.py index 0f14f9f5bf..71a1484de3 100644 --- a/dimos/robot/test_ros_observable_topic.py +++ b/dimos/robot/test_ros_observable_topic.py @@ -67,7 +67,8 @@ def destroy_subscription(self, subscription): # we are doing this in order to avoid importing ROS dependencies if ros tests aren't runnin -def init_robot(): +@pytest.fixture +def robot(): from dimos.robot.ros_observable_topic import ROSObservableTopicAbility class MockRobot(ROSObservableTopicAbility): @@ -87,8 +88,7 @@ def __init__(self): # 4. that the system replays the last message to new observers, # before the new ROS sub starts producing @pytest.mark.ros -def test_parallel_and_cleanup(): - robot = init_robot() +def test_parallel_and_cleanup(robot): from nav_msgs import msg received_messages = [] @@ -160,8 +160,7 @@ def test_parallel_and_cleanup(): # ├──► observe_on(pool) ─► backpressure.latest ─► sub2 (slow) # └──► observe_on(pool) ─► backpressure.latest ─► sub3 (slower) @pytest.mark.ros -def test_parallel_and_hog(): - robot = init_robot() +def test_parallel_and_hog(robot): from nav_msgs import msg obs1 = robot.topic("/odom", msg.Odometry) @@ -201,8 +200,7 @@ def test_parallel_and_hog(): @pytest.mark.asyncio @pytest.mark.ros -async def test_topic_latest_async(): - robot = init_robot() +async def test_topic_latest_async(robot): from nav_msgs import msg odom = await robot.topic_latest_async("/odom", msg.Odometry) @@ -215,16 +213,14 @@ async def test_topic_latest_async(): @pytest.mark.ros -def test_topic_auto_conversion(): - robot = init_robot() +def test_topic_auto_conversion(robot): odom = robot.topic("/vector", Vector).subscribe(lambda x: print(x)) time.sleep(0.5) odom.dispose() @pytest.mark.ros -def test_topic_latest_sync(): - robot = init_robot() +def test_topic_latest_sync(robot): from nav_msgs import msg odom = robot.topic_latest("/odom", msg.Odometry) @@ -237,8 +233,7 @@ def test_topic_latest_sync(): @pytest.mark.ros -def test_topic_latest_sync_benchmark(): - robot = init_robot() +def test_topic_latest_sync_benchmark(robot): from nav_msgs import msg odom = robot.topic_latest("/odom", msg.Odometry) diff --git a/pyproject.toml b/pyproject.toml index 5bab3b2631..2f81fd62b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,4 +37,4 @@ markers = [ "needsdata: needs test data to be downloaded", "ros: depend on ros"] -addopts = "-v -ra --color=yes -m 'not vis and not benchmark and not exclude and not tool and not needsdata'" +addopts = "-v -ra --color=yes -m 'not vis and not benchmark and not exclude and not tool and not needsdata and not ros'" From 23c04321da0fb68bae3d9abea5683c712658d6bd Mon Sep 17 00:00:00 2001 From: lesh Date: Fri, 6 Jun 2025 14:43:41 +0300 Subject: [PATCH 28/30] testing better cache --- .github/workflows/_docker-build-template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_docker-build-template.yml b/.github/workflows/_docker-build-template.yml index d45e88afbc..dd4f3eab66 100644 --- a/.github/workflows/_docker-build-template.yml +++ b/.github/workflows/_docker-build-template.yml @@ -59,6 +59,6 @@ jobs: context: ${{ inputs.context }} file: docker/${{ inputs.dockerfile }}/Dockerfile tags: ${{ inputs.to-image }} - cache-from: type=gha,scope=${{ inputs.dockerfile }} - cache-to: type=gha,mode=max,scope=${{ inputs.dockerfile }} + cache-from: type=gha,scope=${{ inputs.dockerfile }}-${{ inputs.from-image }} + cache-to: type=gha,mode=max,scope=${{ inputs.dockerfile }}-${{ inputs.from-image }} build-args: FROM_IMAGE=${{ inputs.from-image }} From cb989f70ba782a381a6c463d44434dc88f66c8b9 Mon Sep 17 00:00:00 2001 From: lesh Date: Wed, 11 Jun 2025 14:53:00 +0300 Subject: [PATCH 29/30] default args for FROM_IMAGE builds in docker files, nodejs in dev --- .github/workflows/docker.yml | 1 - docker/dev/Dockerfile | 12 ++++++++++-- docker/python/Dockerfile | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 72aba8a359..0de7cc6abe 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,3 @@ -# trigger build: 1 name: docker on: workflow_call diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 2baed56981..d158095042 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM_IMAGE +ARG FROM_IMAGE=ghcr.io/dimensionalos/ros-python:dev FROM ${FROM_IMAGE} ARG GIT_COMMIT=unknown @@ -14,7 +14,9 @@ RUN apt-get install -y \ htop \ python-is-python3 \ iputils-ping \ - wget + wget \ + pre-commit + # Configure git to trust any directory (resolves dubious ownership issues in containers) RUN git config --global --add safe.directory '*' @@ -27,6 +29,12 @@ COPY motd /etc/motd COPY /docker/dev/bash.sh /root/.bash.sh COPY /docker/dev/tmux.conf /root/.tmux.conf +# Install nodejs (for random devtooling like copilot etc) +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash +RUN /root/.nvm/nvm.sh +RUN nvm install node # v24 atm + +# This doesn't work atm RUN echo " v_${GIT_BRANCH}:${GIT_COMMIT} | $(date)" >> /etc/motd RUN echo "echo -e '\033[34m$(cat /etc/motd)\033[0m\n'" >> /root/.bashrc diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile index 86cedbb9f3..f08510faa5 100644 --- a/docker/python/Dockerfile +++ b/docker/python/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM_IMAGE +ARG FROM_IMAGE=ghcr.io/dimensionalos/ros:dev FROM ${FROM_IMAGE} # Install basic requirements From ced737dd2a819340df549d98e4c11755b32c1379 Mon Sep 17 00:00:00 2001 From: lesh Date: Wed, 11 Jun 2025 18:21:45 +0300 Subject: [PATCH 30/30] dockerfile nvm fix --- docker/dev/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index d158095042..2c8c828059 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -31,8 +31,8 @@ COPY /docker/dev/tmux.conf /root/.tmux.conf # Install nodejs (for random devtooling like copilot etc) RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash -RUN /root/.nvm/nvm.sh -RUN nvm install node # v24 atm +ENV NVM_DIR=/root/.nvm +RUN bash -c "source $NVM_DIR/nvm.sh && nvm install 24" # This doesn't work atm RUN echo " v_${GIT_BRANCH}:${GIT_COMMIT} | $(date)" >> /etc/motd