From c9d7ef5f8e0ad169e0252d2c50025ec9fb2aced1 Mon Sep 17 00:00:00 2001 From: The 8472 Date: Sun, 26 Oct 2025 17:56:39 +0100 Subject: [PATCH 1/4] CI: use alternative disks if available cleaning up disk space takes a lot of time --- src/ci/docker/run.sh | 1 + src/ci/scripts/free-disk-space-linux.sh | 46 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 044f5a8fff322..bff1a2217f26c 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -347,6 +347,7 @@ docker \ --env DEPLOY \ --env DEPLOY_ALT \ --env CI \ + --env GIT_DISCOVERY_ACROSS_FILESYSTEM=1 \ --env GITHUB_ACTIONS \ --env GITHUB_REF \ --env GITHUB_STEP_SUMMARY="/checkout/obj/${SUMMARY_FILE}" \ diff --git a/src/ci/scripts/free-disk-space-linux.sh b/src/ci/scripts/free-disk-space-linux.sh index ac3c9cfb28b87..ee831b474df03 100755 --- a/src/ci/scripts/free-disk-space-linux.sh +++ b/src/ci/scripts/free-disk-space-linux.sh @@ -247,12 +247,58 @@ cleanSwap() { free -h } +# Try to find a different drive to put our data on so we don't need to run cleanup. +# The availability of the disks we're probing isn't guaranteed, +# so this is opportunistic. +checkAlternative() { + local gha_alt_disk="/mnt" + + # we need ~50GB of space + local space_target_kb=$((50 * 1024 * 1024)) + local available_space_kb=$(df -k "$gha_alt_disk" --output=avail | tail -n 1) + + # ignore-tidy-linelength + local mntopts="defaults,discard,journal_async_commit,barrier=0,noauto_da_alloc,lazytime,data=writeback" + + # GHA has a 2nd disk mounted at /mnt that is almost empty. + if mountpoint "$gha_alt_disk" && [ "$available_space_kb" -ge "$space_target_kb" ]; then + local blkdev=$(df -k "$gha_alt_disk" --output=source | tail -n 1) + echo "Sufficient space available on $blkdev mounted at $gha_alt_disk" + sudo swapoff -a || true + sudo umount "$gha_alt_disk" + mkdir ./obj + # remount with O_EATMYDATA while we're at it + sudo mount $blkdev ./obj -o $mntopts || (sudo dmesg | tail -n 20 ; exit 1) + sudo chown -R "$USER":"$USER" ./obj + + exit 0 + fi + + # ephemeral NVMe drives on AWS + for dev in /dev/nvme*n1; do + if [ -b "$dev" ] && [ "$(mount | grep "$dev" | wc -l)" -eq 0 ]; then + echo "Found unused block device $dev, creating filesystem" + sudo mkfs.ext4 -E lazy_itable_init=1,lazy_journal_init=1 "$dev" + mkdir ./obj + sudo mount "$dev" ./obj -o $mntopts + sudo chown -R "$USER":"$USER" ./obj + + exit 0 + fi + done +} + + + # Display initial disk space stats AVAILABLE_INITIAL=$(getAvailableSpace) printDF "BEFORE CLEAN-UP:" echo "" + +checkAlternative + execAndMeasureSpaceChange cleanPackages "Unused packages" execAndMeasureSpaceChange cleanDocker "Docker images" execAndMeasureSpaceChange cleanSwap "Swap storage" From 0abb7f4afc530b2c9e99d4ebed42fc61c17926f0 Mon Sep 17 00:00:00 2001 From: the8472 Date: Mon, 27 Oct 2025 23:37:04 +0100 Subject: [PATCH 2/4] [review] add more comments Co-authored-by: Marco Ieni <11428655+marcoieni@users.noreply.github.com> --- src/ci/scripts/free-disk-space-linux.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/ci/scripts/free-disk-space-linux.sh b/src/ci/scripts/free-disk-space-linux.sh index ee831b474df03..d421b4b49c466 100755 --- a/src/ci/scripts/free-disk-space-linux.sh +++ b/src/ci/scripts/free-disk-space-linux.sh @@ -257,25 +257,38 @@ checkAlternative() { local space_target_kb=$((50 * 1024 * 1024)) local available_space_kb=$(df -k "$gha_alt_disk" --output=avail | tail -n 1) + # mount options that trade durability for performance # ignore-tidy-linelength local mntopts="defaults,discard,journal_async_commit,barrier=0,noauto_da_alloc,lazytime,data=writeback" # GHA has a 2nd disk mounted at /mnt that is almost empty. + # Check if it's a valid mountpoint and it has enough available space. if mountpoint "$gha_alt_disk" && [ "$available_space_kb" -ge "$space_target_kb" ]; then local blkdev=$(df -k "$gha_alt_disk" --output=source | tail -n 1) echo "Sufficient space available on $blkdev mounted at $gha_alt_disk" + # see cleanSwap(), swapfile may be mounted under /mnt sudo swapoff -a || true + + # unmount from original location sudo umount "$gha_alt_disk" + + # remount under the obj dir which is used by docker scripts to write most + # of our build output. And apply optimized mount options while we're at it. mkdir ./obj - # remount with O_EATMYDATA while we're at it sudo mount $blkdev ./obj -o $mntopts || (sudo dmesg | tail -n 20 ; exit 1) + + # ensure current user can access everything. + # later scripts assume they have recursive access to obj sudo chown -R "$USER":"$USER" ./obj + # Exit from this script to avoid wasting time removing disk space, + # as we already have enough disk space in the alternative drive. exit 0 fi # ephemeral NVMe drives on AWS for dev in /dev/nvme*n1; do + # check that it's a blockdev and not mounted. if [ -b "$dev" ] && [ "$(mount | grep "$dev" | wc -l)" -eq 0 ]; then echo "Found unused block device $dev, creating filesystem" sudo mkfs.ext4 -E lazy_itable_init=1,lazy_journal_init=1 "$dev" From a5002b5a1ee19ceb3464943c5a85ae9c05409feb Mon Sep 17 00:00:00 2001 From: The 8472 Date: Thu, 13 Nov 2025 22:46:39 +0100 Subject: [PATCH 3/4] print a warning and fallback instead of failing ci we assume the disks are ext4, but GH doesn't guarantee that. --- src/ci/scripts/free-disk-space-linux.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ci/scripts/free-disk-space-linux.sh b/src/ci/scripts/free-disk-space-linux.sh index d421b4b49c466..8a8f15636de2d 100755 --- a/src/ci/scripts/free-disk-space-linux.sh +++ b/src/ci/scripts/free-disk-space-linux.sh @@ -275,7 +275,11 @@ checkAlternative() { # remount under the obj dir which is used by docker scripts to write most # of our build output. And apply optimized mount options while we're at it. mkdir ./obj - sudo mount $blkdev ./obj -o $mntopts || (sudo dmesg | tail -n 20 ; exit 1) + if ! sudo mount $blkdev ./obj -o $mntopts; then + sudo dmesg | tail -n 20 # kernel log should have more details for mount failures + echo "::warning::Failed to remount $blkdev to ./obj with options: $mntopts" + return + fi # ensure current user can access everything. # later scripts assume they have recursive access to obj From 183f29bc3bfbdb9cc8bd5154f7f02730a8a959ad Mon Sep 17 00:00:00 2001 From: The 8472 Date: Mon, 1 Dec 2025 22:41:07 +0100 Subject: [PATCH 4/4] AWS runners have enough disk, no need for cleanup or mount shenanigans --- src/ci/scripts/free-disk-space-linux.sh | 31 ++++++++++++------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/ci/scripts/free-disk-space-linux.sh b/src/ci/scripts/free-disk-space-linux.sh index 8a8f15636de2d..e4e9adc4bc2fd 100755 --- a/src/ci/scripts/free-disk-space-linux.sh +++ b/src/ci/scripts/free-disk-space-linux.sh @@ -4,6 +4,11 @@ set -euo pipefail # Free disk space on Linux GitHub action runners # Script inspired by https://github.com/jlumbroso/free-disk-space + +# we need ~50GB of space +space_target_kb=$((50 * 1024 * 1024)) + + isX86() { local arch arch=$(uname -m) @@ -247,14 +252,21 @@ cleanSwap() { free -h } +sufficientSpaceEarlyExit() { + local available_space_kb=$(df -k . --output=avail | tail -n 1) + + if [ "$available_space_kb" -ge "$space_target_kb" ]; then + echo "Sufficient disk space available (${available_space_kb}KB >= ${space_target_kb}KB). Skipping cleanup." + exit 0 + fi +} + # Try to find a different drive to put our data on so we don't need to run cleanup. # The availability of the disks we're probing isn't guaranteed, # so this is opportunistic. checkAlternative() { local gha_alt_disk="/mnt" - # we need ~50GB of space - local space_target_kb=$((50 * 1024 * 1024)) local available_space_kb=$(df -k "$gha_alt_disk" --output=avail | tail -n 1) # mount options that trade durability for performance @@ -289,20 +301,6 @@ checkAlternative() { # as we already have enough disk space in the alternative drive. exit 0 fi - - # ephemeral NVMe drives on AWS - for dev in /dev/nvme*n1; do - # check that it's a blockdev and not mounted. - if [ -b "$dev" ] && [ "$(mount | grep "$dev" | wc -l)" -eq 0 ]; then - echo "Found unused block device $dev, creating filesystem" - sudo mkfs.ext4 -E lazy_itable_init=1,lazy_journal_init=1 "$dev" - mkdir ./obj - sudo mount "$dev" ./obj -o $mntopts - sudo chown -R "$USER":"$USER" ./obj - - exit 0 - fi - done } @@ -314,6 +312,7 @@ AVAILABLE_INITIAL=$(getAvailableSpace) printDF "BEFORE CLEAN-UP:" echo "" +sufficientSpaceEarlyExit checkAlternative execAndMeasureSpaceChange cleanPackages "Unused packages"