diff --git a/.github/workflows/_docker-build-template.yml b/.github/workflows/_docker-build-template.yml index 48daabe22e..47e99863a5 100644 --- a/.github/workflows/_docker-build-template.yml +++ b/.github/workflows/_docker-build-template.yml @@ -19,14 +19,10 @@ jobs: packages: write steps: - - name: exit early - if: ${{ !inputs.should-run }} - run: | - exit 0 - name: free up disk space # takes a bit of time, so disabled by default # explicitly enable this for large builds - if: ${{ inputs.freespace }} + if: ${{ inputs.should-run && inputs.freespace }} run: | echo -e "pre cleanup space:\n $(df -h)" sudo rm -rf /opt/ghc @@ -35,9 +31,16 @@ jobs: sudo rm -rf /usr/local/lib/android echo -e "post cleanup space:\n $(df -h)" + - name: Fix permissions + if: ${{ inputs.should-run }} + run: | + sudo chown -R $USER:$USER ${{ github.workspace }} || true + - uses: actions/checkout@v4 + if: ${{ inputs.should-run }} - uses: docker/login-action@v3 + if: ${{ inputs.should-run }} with: registry: ghcr.io username: ${{ github.actor }} @@ -45,15 +48,18 @@ jobs: # required for github cache of docker layers - uses: crazy-max/ghaction-github-runtime@v3 + if: ${{ inputs.should-run }} # required for github cache of docker layers - uses: docker/setup-buildx-action@v3 + if: ${{ inputs.should-run }} with: driver: docker-container install: true use: true - uses: docker/build-push-action@v6 + if: ${{ inputs.should-run }} with: push: true context: ${{ inputs.context }} diff --git a/.github/workflows/cleanup-runner.yml b/.github/workflows/cleanup-runner.yml new file mode 100644 index 0000000000..f51c69d638 --- /dev/null +++ b/.github/workflows/cleanup-runner.yml @@ -0,0 +1,47 @@ +name: cleanup-runner + +on: + workflow_run: + workflows: ["docker"] + types: [completed] + workflow_dispatch: + +jobs: + cleanup: + runs-on: [self-hosted, Linux] + steps: + - name: Check disk usage + id: disk-check + run: | + USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') + echo "Disk usage: ${USAGE}%" + echo "usage=${USAGE}" >> $GITHUB_OUTPUT + + - name: Clean Docker images + if: steps.disk-check.outputs.usage > 50 + run: | + echo "=== Docker usage before cleanup ===" + docker system df + + echo -e "\n=== Removing dangling images ===" + docker images -f "dangling=true" -q | xargs -r docker rmi || true + + echo -e "\n=== Docker usage after cleanup ===" + docker system df + + echo -e "\n=== Disk usage after cleanup ===" + df -h / + + - name: Aggressive cleanup if disk critically full + if: steps.disk-check.outputs.usage > 90 + run: | + echo "=== CRITICAL: Disk usage above 90% - Aggressive cleanup ===" + + echo -e "\n=== Removing images older than 3 days ===" + docker image prune -a --filter "until=72h" -f + + echo -e "\n=== Final docker usage ===" + docker system df + + echo -e "\n=== Final disk usage ===" + df -h / \ No newline at end of file diff --git a/.github/workflows/code-cleanup.yml b/.github/workflows/code-cleanup.yml index f00643c75b..d6ff1aec73 100644 --- a/.github/workflows/code-cleanup.yml +++ b/.github/workflows/code-cleanup.yml @@ -10,6 +10,10 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: + - name: Fix permissions + run: | + sudo chown -R $USER:$USER ${{ github.workspace }} || true + - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - name: Run pre-commit diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c7eb4dfca7..0227bba091 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,7 +8,7 @@ permissions: jobs: check-changes: - runs-on: dimos-runner-ubuntu-2204 + runs-on: [self-hosted, Linux] outputs: ros: ${{ steps.filter.outputs.ros }} python: ${{ steps.filter.outputs.python }} @@ -16,6 +16,10 @@ jobs: tests: ${{ steps.filter.outputs.tests }} branch-tag: ${{ steps.set-tag.outputs.branch_tag }} steps: + - name: Fix permissions + run: | + sudo chown -R $USER:$USER ${{ github.workspace }} || true + - uses: actions/checkout@v4 - id: filter uses: dorny/paths-filter@v3 @@ -34,8 +38,6 @@ jobs: - .github/workflows/_docker-build-template.yml - .github/workflows/docker.yml - docker/python/** - - requirements*.txt - - requirements.txt - pyproject.toml dev: @@ -74,6 +76,7 @@ jobs: if: needs.check-changes.outputs.ros == 'true' uses: ./.github/workflows/_docker-build-template.yml with: + should-run: true from-image: ubuntu:22.04 to-image: ghcr.io/dimensionalos/ros:${{ needs.check-changes.outputs.branch-tag }} dockerfile: ros @@ -177,3 +180,45 @@ jobs: }} cmd: "pytest -m heavy" dev-image: dev:${{ needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} + + # TODO: Remove when merge to main as workflow_run needed + cleanup-runner: + needs: [run-tests, run-heavy-tests, run-ros-tests] + if: always() + runs-on: [self-hosted, Linux] + steps: + - name: Check disk usage + id: disk-check + run: | + USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') + echo "Disk usage: ${USAGE}%" + echo "usage=${USAGE}" >> $GITHUB_OUTPUT + + - name: Clean Docker images + if: steps.disk-check.outputs.usage > 50 + run: | + echo "=== Docker usage before cleanup ===" + docker system df + + echo -e "\n=== Removing dangling images ===" + docker images -f "dangling=true" -q | xargs -r docker rmi || true + + echo -e "\n=== Docker usage after cleanup ===" + docker system df + + echo -e "\n=== Disk usage after cleanup ===" + df -h / + + - name: Aggressive cleanup if disk critically full + if: steps.disk-check.outputs.usage > 90 + run: | + echo "=== CRITICAL: Disk usage above 90% - Aggressive cleanup ===" + + echo -e "\n=== Removing images older than 3 days ===" + docker image prune -a --filter "until=72h" -f + + echo -e "\n=== Final docker usage ===" + docker system df + + echo -e "\n=== Final disk usage ===" + df -h / diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 46b8650cfe..a461ea4d6a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,7 +42,10 @@ jobs: image: ghcr.io/dimensionalos/${{ inputs.dev-image }} steps: - + - name: Fix permissions + run: | + sudo chown -R $USER:$USER ${{ github.workspace }} || true + - uses: actions/checkout@v4 - name: Run tests diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index ad81abdbbd..fe002b6d5f 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -22,10 +22,17 @@ RUN apt-get update && apt-get install -y \ # Configure git to trust any directory (resolves dubious ownership issues in containers) RUN git config --global --add safe.directory '*' -COPY . /app/ WORKDIR /app + +COPY pyproject.toml /app/ +RUN mkdir -p /app/dimos +RUN touch /app/dimos/__init__.py + +# Install dependencies RUN --mount=type=cache,target=/root/.cache/pip pip install .[dev] +COPY . /app/ + # Copy files and add version to motd COPY /assets/dimensionalascii.txt /etc/motd COPY /docker/dev/bash.sh /root/.bash.sh diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile index 8689638b0d..67eae50f9c 100644 --- a/docker/python/Dockerfile +++ b/docker/python/Dockerfile @@ -37,10 +37,13 @@ RUN apt-get install -y \ # Fix distutils-installed packages that block pip upgrades RUN apt-get purge -y python3-blinker python3-sympy python3-oauthlib || true -RUN mkdir -p /app/dimos - -COPY . /app/ - WORKDIR /app +COPY pyproject.toml /app/ +RUN mkdir -p /app/dimos +RUN touch /app/dimos/__init__.py + +# Install dependencies RUN --mount=type=cache,target=/root/.cache/pip bash -c "pip install --upgrade 'pip>=24' 'setuptools>=70' 'wheel' 'packaging>=24' && pip install '.[cpu]'" + +COPY . /app/ \ No newline at end of file