diff --git a/.github/workflows/_docker-build-template.yml b/.github/workflows/_docker-build-template.yml index b19be34731..730f4a4696 100644 --- a/.github/workflows/_docker-build-template.yml +++ b/.github/workflows/_docker-build-template.yml @@ -5,7 +5,7 @@ on: from-image: { type: string, required: true } to-image: { type: string, required: true } dockerfile: { type: string, required: true } - freespace: { type: boolean, default: false } + freespace: { type: boolean, default: true } should-run: { type: boolean, default: false } context: { type: string, default: '.' } @@ -19,8 +19,17 @@ jobs: packages: write steps: + - name: Fix permissions + if: ${{ inputs.should-run }} + run: | + sudo chown -R $USER:$USER ${{ github.workspace }} || true + + - uses: actions/checkout@v4 + if: ${{ inputs.should-run }} + with: + fetch-depth: 0 + - name: free up disk space - # takes a bit of time, so disabled by default # explicitly enable this for large builds if: ${{ inputs.should-run && inputs.freespace }} run: | @@ -29,16 +38,84 @@ jobs: sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/share/boost sudo rm -rf /usr/local/lib/android + + echo "=== Cleaning images from deleted branches ===" + + # Get list of all remote branches + git ls-remote --heads origin | awk '{print $2}' | sed 's|refs/heads/||' > /tmp/active_branches.txt + + # Check each docker image tag against branch list + docker images --format "{{.Repository}}:{{.Tag}}|{{.ID}}" | \ + grep "ghcr.io/dimensionalos" | \ + grep -v ":" | \ + while IFS='|' read image_ref id; do + tag=$(echo "$image_ref" | cut -d: -f2) + + # Skip if tag matches an active branch + if grep -qx "$tag" /tmp/active_branches.txt; then + echo "Branch exists: $tag - keeping $image_ref" + else + echo "Branch deleted: $tag - removing $image_ref" + docker rmi "$id" 2>/dev/null || true + fi + done + + rm -f /tmp/active_branches.txt + + USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') + echo "Pre-docker-cleanup disk usage: ${USAGE}%" + + if [ $USAGE -gt 60 ]; then + echo "=== Running quick cleanup (usage > 60%) ===" + + # Keep newest image per tag + docker images --format "{{.Repository}}|{{.Tag}}|{{.ID}}" | \ + grep "ghcr.io/dimensionalos" | \ + grep -v "" | \ + while IFS='|' read repo tag id; do + created_ts=$(docker inspect -f '{{.Created}}' "$id" 2>/dev/null) + created_unix=$(date -d "$created_ts" +%s 2>/dev/null || echo "0") + echo "${repo}|${tag}|${id}|${created_unix}" + done | sort -t'|' -k1,1 -k2,2 -k4,4nr | \ + awk -F'|' ' + { + repo=$1; tag=$2; id=$3 + repo_tag = repo ":" tag + + # Skip protected tags + if (tag ~ /^(main|dev|latest)$/) next + + # Keep newest per tag, remove older duplicates + if (!(repo_tag in seen_combos)) { + seen_combos[repo_tag] = 1 + } else { + system("docker rmi " id " 2>/dev/null || true") + } + }' + + docker image prune -f + docker volume prune -f + fi + + # Aggressive cleanup if still above 85% + USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') + if [ $USAGE -gt 85 ]; then + echo "=== AGGRESSIVE cleanup (usage > 85%) - removing all except main/dev ===" + + # Remove ALL images except main and dev tags + docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}" | \ + grep -E "ghcr.io/dimensionalos" | \ + grep -vE ":(main|dev)$" | \ + awk '{print $2}' | xargs -r docker rmi -f || true + + docker container prune -f + docker volume prune -a -f + docker network prune -f + docker image prune -f + fi + echo -e "post cleanup space:\n $(df -h)" - - name: Fix permissions - if: ${{ inputs.should-run }} - run: | - sudo chown -R $USER:$USER ${{ github.workspace }} || true - - - uses: actions/checkout@v4 - if: ${{ inputs.should-run }} - - uses: docker/login-action@v3 if: ${{ inputs.should-run }} with: diff --git a/.github/workflows/cleanup-runner.yml b/.github/workflows/cleanup-runner.yml deleted file mode 100644 index e0b83ab147..0000000000 --- a/.github/workflows/cleanup-runner.yml +++ /dev/null @@ -1,144 +0,0 @@ -name: cleanup-runner - -on: - workflow_run: - workflows: ["docker"] - types: [completed] - workflow_dispatch: - schedule: - - cron: '0 */6 * * *' # Every 6 hours - -jobs: - cleanup: - runs-on: [self-hosted, Linux] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Check disk usage - id: disk-check - run: | - echo "=== Initial disk usage ===" - df -h / - docker system df - - USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') - echo "Disk usage: ${USAGE}%" - echo "usage=${USAGE}" >> $GITHUB_OUTPUT - - - name: Clean deleted branches - if: always() - run: | - echo "=== Cleaning images from deleted branches ===" - - # Get list of all remote branches - git ls-remote --heads origin | awk '{print $2}' | sed 's|refs/heads/||' > /tmp/active_branches.txt - - # Check each docker image tag against branch list - docker images --format "{{.Repository}}:{{.Tag}}|{{.ID}}" | \ - grep "ghcr.io/dimensionalos" | \ - grep -v ":" | \ - while IFS='|' read image_ref id; do - tag=$(echo "$image_ref" | cut -d: -f2) - - # Skip if tag matches an active branch - if grep -qx "$tag" /tmp/active_branches.txt; then - echo "Branch exists: $tag - keeping $image_ref" - else - echo "Branch deleted: $tag - removing $image_ref" - docker rmi "$id" 2>/dev/null || true - fi - done - - rm -f /tmp/active_branches.txt - - - name: Clean Docker images - Keep newest per tag - if: steps.disk-check.outputs.usage > 50 - run: | - echo "=== Smart cleanup - keeping newest image per tag ===" - - # Group by repository AND tag, keep newest of each - docker images --format "{{.Repository}}|{{.Tag}}|{{.ID}}" | \ - grep "ghcr.io/dimensionalos" | \ - grep -v "" | \ - while IFS='|' read repo tag id; do - created_ts=$(docker inspect -f '{{.Created}}' "$id" 2>/dev/null) - created_unix=$(date -d "$created_ts" +%s 2>/dev/null || echo "0") - echo "${repo}|${tag}|${id}|${created_unix}" - done | sort -t'|' -k1,1 -k2,2 -k4,4nr | \ - awk -F'|' ' - { - repo=$1 - tag=$2 - id=$3 - repo_tag = repo ":" tag - - # Always keep protected tags - if (tag ~ /^(main|dev|latest)$/) { - print "Keeping protected: " repo_tag - next - } - - # For each unique repo:tag combination, keep the newest - if (!(repo_tag in seen_combos)) { - seen_combos[repo_tag] = 1 - print "Keeping newest: " repo_tag - } else { - print "Removing older: " repo_tag " (" id ")" - system("docker rmi " id " 2>/dev/null || true") - } - }' - - # Clean dangling images - docker image prune -f - - # Clean unused volumes - docker volume prune -f - - - name: Moderate cleanup - if: steps.disk-check.outputs.usage > 70 - run: | - echo "=== Moderate cleanup - removing all but main/dev/latest ===" - - # Remove all except main, dev, latest tags - docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}" | \ - grep -E "ghcr.io/dimensionalos" | \ - grep -vE ":(main|dev|latest)$" | \ - awk '{print $2}' | xargs -r docker rmi -f || true - - # Clean all unused volumes - docker volume prune -a -f - - # Clean build cache older than 3 days - docker builder prune -a -f --filter "until=72h" - - - name: Aggressive cleanup - if: steps.disk-check.outputs.usage > 85 - run: | - echo "=== AGGRESSIVE cleanup - removing everything except main/dev ===" - - # Remove ALL images except main and dev tags - docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}" | \ - grep -E "ghcr.io/dimensionalos" | \ - grep -vE ":(main|dev)$" | \ - awk '{print $2}' | xargs -r docker rmi -f || true - - # Clean everything else - docker system prune -a -f --volumes - - - name: Final disk check - if: always() - run: | - echo "=== Final disk usage ===" - df -h / - docker system df - - FINAL_USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') - echo "Final disk usage: ${FINAL_USAGE}%" - - if [ $FINAL_USAGE -gt 90 ]; then - echo "::error::Disk still critically full after cleanup!" - exit 1 - fi \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7a98a9f5fd..929462d8ae 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -67,84 +67,6 @@ jobs: echo "branch tag determined: ${branch_tag}" echo branch_tag="${branch_tag}" >> "$GITHUB_OUTPUT" - pre-build-cleanup: - needs: [check-changes] - runs-on: [self-hosted, Linux] - if: always() - steps: - - name: Fix permissions - run: | - sudo chown -R $USER:$USER ${{ github.workspace }} || true - - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Clean deleted branches - run: | - echo "=== Cleaning images from deleted branches ===" - - # Get list of all remote branches - git ls-remote --heads origin | awk '{print $2}' | sed 's|refs/heads/||' > /tmp/active_branches.txt - - # Check each docker image tag against branch list - docker images --format "{{.Repository}}:{{.Tag}}|{{.ID}}" | \ - grep "ghcr.io/dimensionalos" | \ - grep -v ":" | \ - while IFS='|' read image_ref id; do - tag=$(echo "$image_ref" | cut -d: -f2) - - # Skip if tag matches an active branch - if grep -qx "$tag" /tmp/active_branches.txt; then - echo "Branch exists: $tag - keeping $image_ref" - else - echo "Branch deleted: $tag - removing $image_ref" - docker rmi "$id" 2>/dev/null || true - fi - done - - rm -f /tmp/active_branches.txt - - - name: Quick cleanup if needed - run: | - USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') - echo "Pre-build disk usage: ${USAGE}%" - - if [ $USAGE -gt 80 ]; then - echo "=== Running quick cleanup (usage > 80%) ===" - - # Keep newest image per tag - docker images --format "{{.Repository}}|{{.Tag}}|{{.ID}}" | \ - grep "ghcr.io/dimensionalos" | \ - grep -v "" | \ - while IFS='|' read repo tag id; do - created_ts=$(docker inspect -f '{{.Created}}' "$id" 2>/dev/null) - created_unix=$(date -d "$created_ts" +%s 2>/dev/null || echo "0") - echo "${repo}|${tag}|${id}|${created_unix}" - done | sort -t'|' -k1,1 -k2,2 -k4,4nr | \ - awk -F'|' ' - { - repo=$1; tag=$2; id=$3 - repo_tag = repo ":" tag - - # Skip protected tags - if (tag ~ /^(main|dev|latest)$/) next - - # Keep newest per tag, remove older duplicates - if (!(repo_tag in seen_combos)) { - seen_combos[repo_tag] = 1 - } else { - system("docker rmi " id " 2>/dev/null || true") - } - }' - - docker image prune -f - docker volume prune -f - - echo "Post-cleanup usage: $(df / | awk 'NR==2 {print $5}')" - fi - # just a debugger inspect-needs: needs: [check-changes, ros] @@ -155,7 +77,7 @@ jobs: echo '${{ toJSON(needs) }}' ros: - needs: [check-changes, pre-build-cleanup] + needs: [check-changes] if: needs.check-changes.outputs.ros == 'true' uses: ./.github/workflows/_docker-build-template.yml with: @@ -165,7 +87,7 @@ jobs: dockerfile: ros ros-python: - needs: [check-changes, ros, pre-build-cleanup] + needs: [check-changes, ros] if: always() uses: ./.github/workflows/_docker-build-template.yml with: @@ -178,21 +100,19 @@ jobs: from-image: ghcr.io/dimensionalos/ros:${{ needs.ros.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} to-image: ghcr.io/dimensionalos/ros-python:${{ needs.check-changes.outputs.branch-tag }} dockerfile: python - freespace: true python: - needs: [check-changes, pre-build-cleanup] + needs: [check-changes] if: needs.check-changes.outputs.python == 'true' uses: ./.github/workflows/_docker-build-template.yml with: should-run: true - freespace: true dockerfile: python from-image: ubuntu:22.04 to-image: ghcr.io/dimensionalos/python:${{ needs.check-changes.outputs.branch-tag }} dev: - needs: [check-changes, python, pre-build-cleanup] + needs: [check-changes, python] if: always() uses: ./.github/workflows/_docker-build-template.yml @@ -207,7 +127,7 @@ jobs: dockerfile: dev ros-dev: - needs: [check-changes, ros-python, pre-build-cleanup] + needs: [check-changes, ros-python] if: always() uses: ./.github/workflows/_docker-build-template.yml with: @@ -310,43 +230,3 @@ jobs: # run: | # /entrypoint.sh bash -c "pytest -m module" - # TODO: Remove when merge to main as workflow_run needed - cleanup-runner: - needs: [run-tests, run-heavy-tests, run-ros-tests, run-lcm-tests] - runs-on: [self-hosted, Linux] - steps: - - name: Check disk usage - id: disk-check - run: | - USAGE=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') - echo "Disk usage: ${USAGE}%" - echo "usage=${USAGE}" >> $GITHUB_OUTPUT - - - name: Clean Docker images - if: steps.disk-check.outputs.usage > 50 - run: | - echo "=== Docker usage before cleanup ===" - docker system df - - echo -e "\n=== Removing dangling images ===" - docker images -f "dangling=true" -q | xargs -r docker rmi || true - - echo -e "\n=== Docker usage after cleanup ===" - docker system df - - echo -e "\n=== Disk usage after cleanup ===" - df -h / - - - name: Aggressive cleanup if disk critically full - if: steps.disk-check.outputs.usage > 90 - run: | - echo "=== CRITICAL: Disk usage above 90% - Aggressive cleanup ===" - - echo -e "\n=== Removing images older than 3 days ===" - docker image prune -a --filter "until=72h" -f - - echo -e "\n=== Final docker usage ===" - docker system df - - echo -e "\n=== Final disk usage ===" - df -h /