Full Unitree (Local planner, Explore, SpatialMemory) FakeRTC/WebRTC LCM modules working in self-hosted devcontainer

.github/workflows/_docker-build-template.yml

@@ -19,14 +19,10 @@ jobs:
       packages: write
 
     steps:
-      - name: exit early
-        if: ${{ !inputs.should-run }}
-        run: |
-          exit 0
       - name: free up disk space
         # takes a bit of time, so disabled by default
         # explicitly enable this for large builds
-        if: ${{ inputs.freespace }}
+        if: ${{ inputs.should-run && inputs.freespace }}
         run: |
           echo -e "pre cleanup space:\n $(df -h)"
           sudo rm -rf /opt/ghc
@@ -36,24 +32,29 @@ jobs:
           echo -e "post cleanup space:\n $(df -h)"
 
       - uses: actions/checkout@v4
+        if: ${{ inputs.should-run }}
 
       - uses: docker/login-action@v3
+        if: ${{ inputs.should-run }}
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       # required for github cache of docker layers
       - uses: crazy-max/ghaction-github-runtime@v3
+        if: ${{ inputs.should-run }}
 
       # required for github cache of docker layers
       - uses: docker/setup-buildx-action@v3
+        if: ${{ inputs.should-run }}
         with:
           driver: docker-container
           install: true
           use: true
 
       - uses: docker/build-push-action@v6
+        if: ${{ inputs.should-run }}
         with:
           push: true
           context: ${{ inputs.context }}

.github/workflows/cleanup-runner.yml

@@ -0,0 +1,47 @@
+name: cleanup-runner
+
+on:
+  workflow_run:
+    workflows: ["docker"]
+    types: [completed]
+  workflow_dispatch: 
+
+jobs:
+  cleanup:
+    runs-on: [self-hosted, Linux]
+    steps:
+      - name: Check disk usage
+        id: disk-check
+        run: |
+          USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
+          echo "Disk usage: ${USAGE}%"
+          echo "usage=${USAGE}" >> $GITHUB_OUTPUT
+
+      - name: Clean Docker images
+        if: steps.disk-check.outputs.usage > 50
+        run: |
+          echo "=== Docker usage before cleanup ==="
+          docker system df
+
+          echo -e "\n=== Removing dangling images ==="
+          docker images -f "dangling=true" -q | xargs -r docker rmi || true
+
+          echo -e "\n=== Docker usage after cleanup ==="
+          docker system df
+
+          echo -e "\n=== Disk usage after cleanup ==="
+          df -h /
+
+      - name: Aggressive cleanup if disk critically full
+        if: steps.disk-check.outputs.usage > 90
+        run: |
+          echo "=== CRITICAL: Disk usage above 90% - Aggressive cleanup ==="
+
+          echo -e "\n=== Removing images older than 3 days ==="
+          docker image prune -a --filter "until=72h" -f
+
+          echo -e "\n=== Final docker usage ==="
+          docker system df
+
+          echo -e "\n=== Final disk usage ==="
+          df -h /

.github/workflows/docker.yml

@@ -38,8 +38,7 @@ jobs:
               - .github/workflows/_docker-build-template.yml
               - .github/workflows/docker.yml
               - docker/python/**
-              - requirements*.txt
-              - requirements.txt
+              - pyproject.toml
 
             dev:
               - docker/dev/**
@@ -77,6 +76,7 @@ jobs:
     if: needs.check-changes.outputs.ros == 'true'
     uses: ./.github/workflows/_docker-build-template.yml
     with:
+      should-run: true
       from-image: ubuntu:22.04
       to-image: ghcr.io/dimensionalos/ros:${{ needs.check-changes.outputs.branch-tag }}
       dockerfile:  ros
@@ -180,3 +180,73 @@ jobs:
         }}
       cmd: "pytest -m heavy"
       dev-image: dev:${{ needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
+
+  run-lcm-tests:
+    needs: [check-changes, dev]
+    if: always()
+    uses: ./.github/workflows/tests.yml
+    with:
+      should-run: ${{
+        needs.check-changes.result == 'success' &&
+        ((needs.dev.result == 'success') ||
+        (needs.dev.result == 'skipped' &&
+        needs.check-changes.outputs.tests == 'true'))
+        }}
+      cmd: "pytest -m lcm"
+      dev-image: dev:${{ needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
+
+  # Run module tests directly to avoid pytest forking issues
+  run-module-tests:
+    needs: [check-changes, dev]
+    if: always()
+    uses: ./.github/workflows/tests.yml
+    with:
+      should-run: ${{
+        needs.check-changes.result == 'success' &&
+        ((needs.dev.result == 'success') ||
+        (needs.dev.result == 'skipped' &&
+        needs.check-changes.outputs.tests == 'true'))
+        }}
+      cmd: "export PYTHONPATH=$(pwd):$PYTHONPATH && python dimos/perception/test_spatial_memory_module.py"
+      dev-image: dev:${{ needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
+  # TODO: Remove when merge to main as workflow_run needed
+  cleanup-runner:
+    needs: [run-tests, run-heavy-tests, run-ros-tests]
+    if: always()
+    runs-on: [self-hosted, Linux]
+    steps:
+      - name: Check disk usage
+        id: disk-check
+        run: |
+          USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
+          echo "Disk usage: ${USAGE}%"
+          echo "usage=${USAGE}" >> $GITHUB_OUTPUT
+
+      - name: Clean Docker images
+        if: steps.disk-check.outputs.usage > 50
+        run: |
+          echo "=== Docker usage before cleanup ==="
+          docker system df
+
+          echo -e "\n=== Removing dangling images ==="
+          docker images -f "dangling=true" -q | xargs -r docker rmi || true
+
+          echo -e "\n=== Docker usage after cleanup ==="
+          docker system df
+
+          echo -e "\n=== Disk usage after cleanup ==="
+          df -h /
+
+      - name: Aggressive cleanup if disk critically full
+        if: steps.disk-check.outputs.usage > 90
+        run: |
+          echo "=== CRITICAL: Disk usage above 90% - Aggressive cleanup ==="
+
+          echo -e "\n=== Removing images older than 3 days ==="
+          docker image prune -a --filter "until=72h" -f
+
+          echo -e "\n=== Final docker usage ==="
+          docker system df
+
+          echo -e "\n=== Final disk usage ==="
+          df -h /

.github/workflows/tests.yml

@@ -47,10 +47,18 @@ jobs:
           sudo chown -R $USER:$USER ${{ github.workspace }} || true
 
       - uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Configure Git LFS
+        run: |
+          git config --global --add safe.directory '*'
+          git lfs install
+          git lfs fetch
+          git lfs checkout
 
       - name: Run tests
         run: |
-          git config --global --add safe.directory '*'
           /entrypoint.sh bash -c "${{ inputs.cmd }}"
 
       - name: check disk space

dimos/perception/spatial_perception.py

@@ -38,6 +38,8 @@
 
 logger = setup_logger("dimos.perception.spatial_memory")
 
+print("")
+
 
 class SpatialMemory(Module):
     """
@@ -170,7 +172,7 @@ def __init__(
         # Track latest data for processing
         self._latest_video_frame: Optional[np.ndarray] = None
         self._latest_odom: Optional[Odometry] = None
-        self._process_interval = 0.1  # Process at 10Hz
+        self._process_interval = 1
 
         logger.info(f"SpatialMemory initialized with model {embedding_model}")
 

dimos/perception/test_spatial_memory_module.py

@@ -34,6 +34,7 @@
 from dimos.utils.data import get_data
 from dimos.utils.testing import TimedSensorReplay
 from dimos.utils.logging_config import setup_logger
+from unittest.mock import patch, MagicMock
 import warnings
 
 logger = setup_logger("test_spatial_memory_module")
@@ -61,7 +62,8 @@ def start(self):
         self._subscription = (
             video_replay.stream()
             .pipe(
-                ops.sample(0.1)  # Limit to 10 FPS for testing
+                ops.sample(2),  # Sample every 2 seconds for resource-constrained systems
+                ops.take(5),  # Only take 5 frames total
             )
             .subscribe(self.video_out.publish)
         )
@@ -94,7 +96,14 @@ def start(self):
         odom_replay = TimedSensorReplay(self.odom_path, autocast=Odometry.from_msg)
 
         # Subscribe to the replay stream and publish to LCM
-        self._subscription = odom_replay.stream().subscribe(self.odom_out.publish)
+        self._subscription = (
+            odom_replay.stream()
+            .pipe(
+                ops.sample(0.5),  # Sample every 500ms
+                ops.take(10),  # Only take 10 odometry updates total
+            )
+            .subscribe(self.odom_out.publish)
+        )
 
         logger.info("OdometryReplayModule started")
 
@@ -107,7 +116,7 @@ def stop(self):
         logger.info("OdometryReplayModule stopped")
 
 
-@pytest.mark.heavy
+@pytest.mark.skip(reason="Run directly with python")
 class TestSpatialMemoryModule:
     @pytest.fixture(scope="function")
     def temp_dir(self):
@@ -167,7 +176,7 @@ async def test_spatial_memory_module_with_replay(self, temp_dir):
 
             # Wait for some frames to be processed
             logger.info("Waiting for frames to be processed...")
-            await asyncio.sleep(5)  # Process for 5 seconds
+            await asyncio.sleep(3)
 
             # Stop the replay modules to prevent infinite streaming
             video_module.stop()
@@ -210,4 +219,8 @@ async def test_spatial_memory_module_with_replay(self, temp_dir):
 
 
 if __name__ == "__main__":
-    pytest.main(["-v", "-s", __file__])
+    # pytest.main(["-v", "-s", __file__])
+    test = TestSpatialMemoryModule()
+    asyncio.run(
+        test.test_spatial_memory_module_with_replay(tempfile.mkdtemp(prefix="spatial_memory_test_"))
+    )