Fix python and dev docker builds for CICD

.github/workflows/_docker-build-template.yml

@@ -19,14 +19,10 @@ jobs:
       packages: write
 
     steps:
-      - name: exit early
-        if: ${{ !inputs.should-run }}
-        run: |
-          exit 0
       - name: free up disk space
         # takes a bit of time, so disabled by default
         # explicitly enable this for large builds
-        if: ${{ inputs.freespace }}
+        if: ${{ inputs.should-run && inputs.freespace }}
         run: |
           echo -e "pre cleanup space:\n $(df -h)"
           sudo rm -rf /opt/ghc
@@ -35,25 +31,35 @@ jobs:
           sudo rm -rf /usr/local/lib/android
           echo -e "post cleanup space:\n $(df -h)"
 
+      - name: Fix permissions
+        if: ${{ inputs.should-run }}
+        run: |
+          sudo chown -R $USER:$USER ${{ github.workspace }} || true
+
       - uses: actions/checkout@v4
+        if: ${{ inputs.should-run }}
 
       - uses: docker/login-action@v3
+        if: ${{ inputs.should-run }}
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       # required for github cache of docker layers
       - uses: crazy-max/ghaction-github-runtime@v3
+        if: ${{ inputs.should-run }}
 
       # required for github cache of docker layers
       - uses: docker/setup-buildx-action@v3
+        if: ${{ inputs.should-run }}
         with:
           driver: docker-container
           install: true
           use: true
 
       - uses: docker/build-push-action@v6
+        if: ${{ inputs.should-run }}
         with:
           push: true
           context: ${{ inputs.context }}

.github/workflows/cleanup-runner.yml

@@ -0,0 +1,47 @@
+name: cleanup-runner
+
+on:
+  workflow_run:
+    workflows: ["docker"]
+    types: [completed]
+  workflow_dispatch: 
+
+jobs:
+  cleanup:
+    runs-on: [self-hosted, Linux]
+    steps:
+      - name: Check disk usage
+        id: disk-check
+        run: |
+          USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
+          echo "Disk usage: ${USAGE}%"
+          echo "usage=${USAGE}" >> $GITHUB_OUTPUT
+
+      - name: Clean Docker images
+        if: steps.disk-check.outputs.usage > 50
+        run: |
+          echo "=== Docker usage before cleanup ==="
+          docker system df
+
+          echo -e "\n=== Removing dangling images ==="
+          docker images -f "dangling=true" -q | xargs -r docker rmi || true
+
+          echo -e "\n=== Docker usage after cleanup ==="
+          docker system df
+
+          echo -e "\n=== Disk usage after cleanup ==="
+          df -h /
+
+      - name: Aggressive cleanup if disk critically full
+        if: steps.disk-check.outputs.usage > 90
+        run: |
+          echo "=== CRITICAL: Disk usage above 90% - Aggressive cleanup ==="
+
+          echo -e "\n=== Removing images older than 3 days ==="
+          docker image prune -a --filter "until=72h" -f
+
+          echo -e "\n=== Final docker usage ==="
+          docker system df
+
+          echo -e "\n=== Final disk usage ==="
+          df -h /

.github/workflows/code-cleanup.yml

@@ -10,6 +10,10 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
+    - name: Fix permissions
+      run: |
+        sudo chown -R $USER:$USER ${{ github.workspace }} || true
+
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v3
     - name: Run pre-commit

.github/workflows/docker.yml

@@ -8,14 +8,18 @@ permissions:
 
 jobs:
   check-changes:
-    runs-on: dimos-runner-ubuntu-2204
+    runs-on: [self-hosted, Linux]
     outputs:
       ros:    ${{ steps.filter.outputs.ros }}
       python: ${{ steps.filter.outputs.python }}
       dev:    ${{ steps.filter.outputs.dev }}
       tests:  ${{ steps.filter.outputs.tests }}
       branch-tag: ${{ steps.set-tag.outputs.branch_tag }}
     steps:
+      - name: Fix permissions
+        run: |
+          sudo chown -R $USER:$USER ${{ github.workspace }} || true
+
       - uses: actions/checkout@v4
       - id: filter
         uses: dorny/paths-filter@v3
@@ -34,8 +38,6 @@ jobs:
               - .github/workflows/_docker-build-template.yml
               - .github/workflows/docker.yml
               - docker/python/**
-              - requirements*.txt
-              - requirements.txt
               - pyproject.toml
 
             dev:
@@ -74,6 +76,7 @@ jobs:
     if: needs.check-changes.outputs.ros == 'true'
     uses: ./.github/workflows/_docker-build-template.yml
     with:
+      should-run: true
       from-image: ubuntu:22.04
       to-image: ghcr.io/dimensionalos/ros:${{ needs.check-changes.outputs.branch-tag }}
       dockerfile:  ros
@@ -177,3 +180,45 @@ jobs:
         }}
       cmd: "pytest -m heavy"
       dev-image: dev:${{ needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
+
+  # TODO: Remove when merge to main as workflow_run needed
+  cleanup-runner:
+    needs: [run-tests, run-heavy-tests, run-ros-tests]
+    if: always()
+    runs-on: [self-hosted, Linux]
+    steps:
+      - name: Check disk usage
+        id: disk-check
+        run: |
+          USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
+          echo "Disk usage: ${USAGE}%"
+          echo "usage=${USAGE}" >> $GITHUB_OUTPUT
+
+      - name: Clean Docker images
+        if: steps.disk-check.outputs.usage > 50
+        run: |
+          echo "=== Docker usage before cleanup ==="
+          docker system df
+
+          echo -e "\n=== Removing dangling images ==="
+          docker images -f "dangling=true" -q | xargs -r docker rmi || true
+
+          echo -e "\n=== Docker usage after cleanup ==="
+          docker system df
+
+          echo -e "\n=== Disk usage after cleanup ==="
+          df -h /
+
+      - name: Aggressive cleanup if disk critically full
+        if: steps.disk-check.outputs.usage > 90
+        run: |
+          echo "=== CRITICAL: Disk usage above 90% - Aggressive cleanup ==="
+
+          echo -e "\n=== Removing images older than 3 days ==="
+          docker image prune -a --filter "until=72h" -f
+
+          echo -e "\n=== Final docker usage ==="
+          docker system df
+
+          echo -e "\n=== Final disk usage ==="
+          df -h /

.github/workflows/tests.yml

@@ -42,7 +42,10 @@ jobs:
       image: ghcr.io/dimensionalos/${{ inputs.dev-image }}
 
     steps:
-
+      - name: Fix permissions
+        run: |
+          sudo chown -R $USER:$USER ${{ github.workspace }} || true
+
       - uses: actions/checkout@v4
 
       - name: Run tests

docker/dev/Dockerfile

@@ -22,10 +22,17 @@ RUN apt-get update && apt-get install -y \
 # Configure git to trust any directory (resolves dubious ownership issues in containers)
 RUN git config --global --add safe.directory '*'
 
-COPY . /app/
 WORKDIR /app
+
+COPY pyproject.toml /app/
+RUN mkdir -p /app/dimos
+RUN touch /app/dimos/__init__.py
+
+# Install dependencies
 RUN --mount=type=cache,target=/root/.cache/pip pip install .[dev]
 
+COPY . /app/
+
 # Copy files and add version to motd
 COPY /assets/dimensionalascii.txt /etc/motd
 COPY /docker/dev/bash.sh /root/.bash.sh

docker/python/Dockerfile

@@ -37,10 +37,13 @@ RUN apt-get install -y \
 # Fix distutils-installed packages that block pip upgrades
 RUN apt-get purge -y python3-blinker python3-sympy python3-oauthlib || true
 
-RUN mkdir -p /app/dimos
-
-COPY . /app/
-
 WORKDIR /app
 
+COPY pyproject.toml /app/
+RUN mkdir -p /app/dimos
+RUN touch /app/dimos/__init__.py
+
+# Install dependencies
 RUN --mount=type=cache,target=/root/.cache/pip bash -c "pip install --upgrade 'pip>=24' 'setuptools>=70' 'wheel' 'packaging>=24' && pip install '.[cpu]'"
+
+COPY . /app/