Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
171 commits
Select commit Hold shift + click to select a range
d1cb8e8
Port WrenchComposer dual-buffer fix to develop branch (#5265)
AntoineRichard Apr 22, 2026
09fe938
Wires up teleop control states via Isaac Teleop message channel (#5268)
rwiltz Apr 22, 2026
481d7ee
Rendering correctness test determinism (#5353)
huidongc Apr 22, 2026
0a9e45c
Adds presets to environment docs and fix doc build issues (#5360)
kellyguo11 Apr 23, 2026
33f8f3a
Refactors Newton XformPrimView: proper local poses, warp-native API, …
ooctipus Apr 23, 2026
0c565d6
Caches resolve_matching_names on AssetBase for all finder methods (#5…
hougantc-nvda Apr 23, 2026
a0e07b4
Forbid pushing to origin in AGENTS.md (#5344)
AntoineRichard Apr 23, 2026
b655176
Add render flag to environment step() API (#5349)
kellyguo11 Apr 23, 2026
63da1f4
Improve Partial Visuallization (#5347)
matthewtrepte Apr 23, 2026
ae41e2a
Cleans up failing unit tests (#5385)
kellyguo11 Apr 24, 2026
17d2f66
Hydra SDG rendering fix (#5384)
pbarejko Apr 24, 2026
cfdbe71
Minimal Shading Mode (#5346)
pbarejko Apr 24, 2026
ca84d35
Fixes compatibility issues with latest Isaac Sim updates (#5358)
kellyguo11 Apr 25, 2026
4af4969
Update deformable showroom image (#5392)
mmichelis Apr 25, 2026
1d916c3
Migrated golden images to Git LFS (#5361)
huidongc Apr 25, 2026
b69279d
Fixes Docker Image Publishing (#5379)
myurasov-nv Apr 25, 2026
80e48f7
[Exp] Cherry-pick warp MDP migration and capture safety from dev/newt…
hujc7 Apr 25, 2026
b674671
Improves Isaac Lab Mimic tutorial documentation and flow (#5283)
peterd-NV Apr 25, 2026
080898f
Updates deprecated extensions in Isaac Sim (#5293)
kellyguo11 Apr 26, 2026
4d58ce4
Updates tests for Isaac Sim CI pipeline (#5403)
kellyguo11 Apr 26, 2026
d76c153
Add checked_apply helper for forwarding configclass fields onto upstr…
hujc7 Apr 26, 2026
1a040c0
[Newton] Add Rough terrain locomotion Part 1: Foundation + quadrupeds…
hujc7 Apr 26, 2026
544f7b1
Fixes arm64 docker build failure (#5408)
myurasov-nv Apr 27, 2026
01014fe
ProxyArray and Asset/Sensor level property caching (#5304)
AntoineRichard Apr 27, 2026
d1a6cdb
Fixes arm64 docker build failure
myurasov-nv Apr 27, 2026
301094d
Runtime switch to read transforms from GPU. (#5411)
pbarejko Apr 27, 2026
4085387
Adds more published docker image tags (#5417)
myurasov-nv Apr 28, 2026
640e71b
Adds initial support for the Kamino solver (#5214)
AntoineRichard Apr 28, 2026
555e3ed
Fix Newton mock missing articulation_ids and run all CI tests on ever…
nvsekkin Apr 29, 2026
83a1d81
Updated TiledCamera references to Camera in downstream configs (#5377)
bdilinila Apr 29, 2026
ad399b8
ci: Improve Docker cleanup for test jobs (#5438)
AntoineRichard Apr 29, 2026
12d935b
Decouple Camera from RTX/Replicator output-buffer logic (#5366)
nvsekkin Apr 29, 2026
15dd6a1
Add Metrics/success_rate logging across RL tasks (#5415)
ooctipus Apr 30, 2026
f0234a8
Fix camera obs hang when xr is enabled (#5425)
rwiltz Apr 30, 2026
4f85aae
Merges updates from main (#5436)
kellyguo11 Apr 30, 2026
ab0a141
Pin PyTorch intersphinx to /docs/2.11/ to fix broken /stable/ alias (…
nvsekkin Apr 30, 2026
f267f96
CI: reliable pipeline cancellation (per-PR concurrency + interruptibl…
nvsekkin Apr 30, 2026
e57269f
Performance optimization on asset writes (#5329)
AntoineRichard Apr 30, 2026
951978b
Gate isaacteleop dependency on Linux to fix Windows install (#5441)
rwiltz Apr 30, 2026
9817fce
Fix required CI checks for docs-only PRs (#5454)
AntoineRichard Apr 30, 2026
88d2407
OVRTX Correctness (#5401)
huidongc Apr 30, 2026
41908c0
Makes ovphysx an optional install of isaaclab_ovphysx (#5428)
ooctipus Apr 30, 2026
5037857
Avoids force push for doc deploy job (#5460)
kellyguo11 Apr 30, 2026
082e488
Fixes more installation issues for ARM and Windows (#5431)
myurasov-nv Apr 30, 2026
8fdb110
Remove colors of deformable cube tutorial docs (#5456)
mmichelis Apr 30, 2026
61c8a94
Fixes multi-GPU device resolution in AppLauncher (#5215)
kellyguo11 May 1, 2026
3827956
[Rough Locomotion] Part 2: H1/Cassie bipeds on Newton (#5298)
hujc7 May 1, 2026
11b8c64
Suspends Fabric USD notice listener during cloning for faster startup…
ooctipus May 1, 2026
a6e7577
Document Kamino solver presets (#5457)
AntoineRichard May 1, 2026
6dfa5fa
OMPE-90534: Add json nsys trace definitions (#5397)
jmart-nv May 1, 2026
2b49dac
Updates wheel versioning (#5467)
myurasov-nv May 1, 2026
3303bef
Adds compatibility note regarding Isaac Sim (#5465)
kellyguo11 May 1, 2026
8bb6eff
Adds retry logic for timeout tests (#5448)
kellyguo11 May 1, 2026
7166229
Skip cloner perf test and fix ray caster intrinsics camera pose (#5470)
ooctipus May 1, 2026
261d077
Patch newton model reqs (#5398)
matthewtrepte May 1, 2026
779a0fc
Adds support for gear insertion with Flexiv Rizon 4s (#5175)
shauryadNv May 2, 2026
9bf2920
Add fragment-based changelog system to eliminate per-PR merge conflic…
hujc7 May 2, 2026
26c253c
CI: trigger build.yaml workflow on scripts/ changes (#5442)
jmart-nv May 3, 2026
d3ce0b3
OMPE-85851: Added success metric tracking to benchmark scripts. (#5335)
jmart-nv May 3, 2026
8394807
[Newton] Adds joint wrench sensor (#5412)
camevor May 3, 2026
d48075a
Adds LEAPP export integration (#5105)
frlai May 3, 2026
26cfdda
Decouple Renderer from Camera (#5297)
r-schmitt May 4, 2026
add60a1
Simplifies CI change detection and concurrency groups (#5466)
myurasov-nv May 4, 2026
9da0c2f
Refactors visualizer cloning around ClonePlan (#5484)
ooctipus May 4, 2026
347ce94
[Rough Locomotion] Part 3: G1 on Newton (max_iterations 3000→5000, no…
hujc7 May 4, 2026
08dbf1f
add RenderContext to docs (#5489)
r-schmitt May 4, 2026
744e371
omniverseclient pin (#5487)
huidongc May 5, 2026
33f5f71
Update a set of small documentation inconsistencies (#5490)
Alex-Omar-Nvidia May 5, 2026
5afbca4
Reduce rendering test flakiness (#5475)
huidongc May 5, 2026
4758900
Restores GR1T2 legacy teleop pipeline for teleop CI (#5254)
rwiltz May 5, 2026
9ffafbb
Replace add_usd with open_usd (#5491)
pbarejko May 5, 2026
2644c1e
Sync video recorder's (--video) renderer backend with the active visu…
matthewtrepte May 5, 2026
7361ece
Add PhysX joint wrench sensor (#5458)
AntoineRichard May 5, 2026
530fa8b
Rename Newton preset to MJWarp (#5488)
AntoineRichard May 5, 2026
632d05f
Refactor NewtonManager into solver-specific subclasses (#5439)
mmichelis May 5, 2026
c0ae5ee
Updating docs, explicitly call out physicx support and handtracking r…
rwiltz May 5, 2026
7a4b6d2
Add Debug Visualization Markers to Newton Based Visualizers (#5473)
matthewtrepte May 5, 2026
fa5959f
Improves hanging flakiness in CI tests (#5479)
kellyguo11 May 5, 2026
2d52d62
Docs: update contributing.rst and PR template for changelog fragments…
hujc7 May 5, 2026
bf646bb
mark all rtx-based rendering tests flaky for now (#5508)
huidongc May 6, 2026
efd9d1e
perf: add PrepareForReuse to FabricFrameView, remove sync_usd_on_fabr…
pv-nvidia May 6, 2026
b258e87
Update pytorch3d installation command in locomanipulation SDG documen…
jaybdub May 6, 2026
7b44452
use RendererCfg as default renderer_cfg in CameraCfg (#5521)
r-schmitt May 7, 2026
b582dab
Refactors schema cfgs to separate solver-common from PhysX-specific f…
vidurv-nvidia May 7, 2026
2388178
Updates docs for using nurec background in locomanipulation sdg (#5301)
dengyuchenkit May 7, 2026
a7514be
[CI][Auto Version Bump] Compile changelog fragments (workflow_dispatch)
isaaclab-bot[bot] May 8, 2026
9e4e62c
[Newton] Bump Newton pin to v1.2.0rc2 (#5523)
hujc7 May 8, 2026
99b0359
[CI][Auto Version Bump] Compile changelog fragments (schedule)
isaaclab-bot[bot] May 8, 2026
af9c98f
Fixes joint friction API docs (#5533)
AntoineRichard May 8, 2026
8c5e2ad
Deprecates state properties calls (#5423)
AntoineRichard May 8, 2026
311ec73
Fixes backend deprecation warning call sites (#5418)
AntoineRichard May 8, 2026
3d93e84
Updates core deprecation call sites (#5409)
AntoineRichard May 8, 2026
a64fcf1
Adds PVA and joint wrench sensor docs (#5532)
AntoineRichard May 8, 2026
a784ed9
[CI unblock] Skip viewergl-fully-black test + fix warp intersphinx 40…
hujc7 May 8, 2026
21a7919
Replicates fk invalidation on other assets (#5367)
AntoineRichard May 8, 2026
513a017
Updates task deprecation call sites (#5410)
AntoineRichard May 8, 2026
e15b1d0
Enabled OVRTX rendering tests on CI (#5492)
huidongc May 8, 2026
24f2cc0
Fix OvPhysX 0.4 compatibility (#5545)
marcodiiga May 8, 2026
65e5ead
Add Kamino solver tutorial (#5483)
AntoineRichard May 8, 2026
23ababb
[Visualizers] Fix viewergl fully-black: assign PyVec3 to Newton camer…
hujc7 May 8, 2026
1336acd
Adds multirotor vision-based navigation task and acceleration, veloci…
Zwoelf12 May 8, 2026
8d9af88
Fixes various installation bugs (#5530)
myurasov-nv May 8, 2026
4687c34
Replaces fcntl with filelock to avoid windows import issues (#5544)
AntoineRichard May 8, 2026
1e0e379
Disables articulation tests in Isaac Sim CI (#5557)
kellyguo11 May 9, 2026
39a9aec
Initialize Warp runtime (#5548)
pbarejko May 9, 2026
b59b3ae
[CI][Auto Version Bump] Compile changelog fragments (schedule)
isaaclab-bot[bot] May 9, 2026
4934cd0
Fix Pink IK DAQP dependency checks (#5556)
ooctipus May 9, 2026
63b1257
[CI][Auto Version Bump] Compile changelog fragments (schedule)
isaaclab-bot[bot] May 10, 2026
6dedbb7
Refactor cloning around cfg-driven ClonePlan (#5528)
ooctipus May 10, 2026
4feb184
[CI][Auto Version Bump] Compile changelog fragments (schedule)
isaaclab-bot[bot] May 11, 2026
e1fba6e
Fixes benchmark scripts when tensorboard logs are missing (#5564)
kellyguo11 May 11, 2026
5442de1
Fixes manus docs moving. (#5577)
hougantc-nvda May 11, 2026
707e87d
Pre and post physics renderer initialization (#5573)
pbarejko May 12, 2026
a337c0b
Enables pipelined IsaacTeleop retargeting (#5493)
hougantc-nvda May 12, 2026
dd5a21a
Add assemble_trocar task for G129-Dex3 (#5082)
mingxueg-nv May 12, 2026
d9bdc7f
Refactors the doc on Schema for Lab 3.0, and adds MuJoCo gravcomp (#5…
vidurv-nvidia May 12, 2026
1f35b18
[CI][Auto Version Bump] Compile changelog fragments (schedule)
isaaclab-bot[bot] May 12, 2026
a2a5167
Fix broken external links flagged by lychee (#5576)
ooctipus May 12, 2026
6621d49
Stop logging spurious carb null-client error from cloner / cubric hel…
ooctipus May 12, 2026
965136d
Adds heterogeneous dexsuite to Newton backend (#5024)
ooctipus May 12, 2026
d58e3d7
Pins Isaac Sim image to previous image from 05/11 (#5600)
kellyguo11 May 13, 2026
b8fead4
[CI][Auto Version Bump] Compile changelog fragments (schedule)
isaaclab-bot[bot] May 13, 2026
5ea7512
[Newton] Fixes contact sensor metadata access (#5588)
camevor May 13, 2026
68a651f
[OVPHYSX] RigidObject + RigidObjectData asset (#5426)
AntoineRichard May 13, 2026
3ece85b
[Newton] Backend-agnostic task-space accessors for IK/OSC (#5400)
hujc7 May 13, 2026
a44eefc
Disables test timeout retry (#5602)
kellyguo11 May 13, 2026
4c99765
Fixes doc build errors in mimic datagen (#5606)
kellyguo11 May 13, 2026
b2582a4
Update configs to new conventions from rsl_rl >= 5.0 (#5551)
StafaH May 13, 2026
ae453e8
New Scene Data Provider (#5128)
daniela-hase May 14, 2026
00ec13e
Restore RT2 RenderCfg fields removed in PhysicsManager refactor (#5167)
nblauch May 14, 2026
a7c791e
Fix LeappDeploymentEnv missing extras attribute (#5560)
klakhi May 14, 2026
d12cfce
Fixes some nvbugs (#5584)
myurasov-nv May 14, 2026
ce95adb
[Newton] Drop explicit mujoco/mujoco-warp pins, defer to newton[sim] …
hujc7 May 14, 2026
0c49777
[Newton] Add Shadow-Hand-Over MAPPO Newton backend (depends on #5433)…
hujc7 May 14, 2026
b65a1ac
[CI][Auto Version Bump] Compile changelog fragments (schedule)
isaaclab-bot[bot] May 14, 2026
ac538fd
Fix leapp docs (#5512)
frlai May 14, 2026
98d4bba
[Changelog] Fix orphan paragraph in 5.2.0 entry and gate future fragm…
hujc7 May 14, 2026
e0a217d
OMPE-92490: Fix singular rotation matrix and non-rotation quaternion …
jmart-nv May 14, 2026
88d3dda
[CI][Auto Version Bump] Compile changelog fragments (schedule)
isaaclab-bot[bot] May 15, 2026
a5eb9ad
Fixes OmniHub startup in Docker tests (#5633)
AntoineRichard May 15, 2026
ef6c178
Fix skrl drone-ARL configs using STATES instead of OBSERVATIONS (#5613)
pascal-roth May 15, 2026
4e62558
Add teleop replay agent for non-interactive CI runs (#5507)
rwiltz May 15, 2026
4985855
Fixed rlinf install docs (#5639)
mingxueg-nv May 15, 2026
7015be2
Refactor train/play and create uv run workflow without dedicated virt…
StafaH May 15, 2026
18c6bf3
[Newton] Rename per-env labels in physics replication (depends on #55…
hujc7 May 15, 2026
94c0967
Updates URDF/MJCF importer to use the latest Isaac Sim importer (#5394)
stevfeng May 15, 2026
112f3c0
[PresetCLI] Add typed preset selection with task-aware help (#5587)
hujc7 May 15, 2026
aef3dbc
[Newton] Bump Newton pin to v1.2.0 (stable) (#5616)
hujc7 May 15, 2026
d84b905
Add patch to locomanipulation SDG pipeline to avoid use of flash attn…
jaybdub May 15, 2026
21745e2
Fix silent ABI mismatch in cubric Python shim (#5444)
jmart-nv May 15, 2026
2bdccb2
Fix invalid inline comment in Windows batch code block (#5612)
klakhi May 15, 2026
bb9388e
Add settings to make training results deterministic (#5449)
redfler May 15, 2026
4737154
Fix OVRTX renderer device mismatch on multi-GPU (#5594)
fatimaanes May 15, 2026
9989d27
Enable OVRTX cloning by default (#5591)
huidongc May 15, 2026
3d002c8
[Fix] Use matrix multiplication for rotation composition in test_pink…
hujc7 May 16, 2026
b541035
[Docs] Expand Isaac Teleop XR performance optimization guide (#5643)
rwiltz May 16, 2026
32462c0
Make Isaac Lab Docker images run as non-root (#5618)
sheikh-nv May 16, 2026
cafc911
Add frame stacking support for explicit temporal info (#5574)
jmart-nv May 16, 2026
3a34736
Migrate camera/renderer/camera data to warp (#5578)
StafaH May 16, 2026
58f633c
[CI][Auto Version Bump] Compile changelog fragments (schedule)
isaaclab-bot[bot] May 16, 2026
6e3482d
[OVPHYSX] Articulation rewrite (data class + asset class + kernels) (…
AntoineRichard May 16, 2026
bed2bf9
Fix lazy import for configclass and provide upper bound for python (#…
pbarejko May 16, 2026
9297faf
Avoid disk I/O when preparing USD stage for OVRTX renderer (#5631)
huidongc May 16, 2026
c67ed38
Move rendering-correctness tests into dedicated CI jobs (#5649)
nvsekkin May 16, 2026
f8d167c
Test run time improvement: remove AppLauncher from tests that do not…
pbarejko May 16, 2026
0dd2412
Split RayCaster into backend-specific implementations (#5510)
ooctipus May 17, 2026
4f0e8c8
Updates ecosystem docs (#5581)
kellyguo11 May 17, 2026
9da557a
[CI][Auto Version Bump] Compile changelog fragments (schedule)
isaaclab-bot[bot] May 17, 2026
99d5097
Expose missing API docs (#5582)
kellyguo11 May 17, 2026
2ace55b
Reworks modular installation and add tests and updates docs (#5650)
kellyguo11 May 17, 2026
8b5f661
refactor: fuse set_world_poses/set_scales into single kernel launch
May 17, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,7 @@
*.jit filter=lfs diff=lfs merge=lfs -text
*.hdf5 filter=lfs diff=lfs merge=lfs -text

source/isaaclab_tasks/test/golden_images/**/*.png filter=lfs diff=lfs merge=lfs -text

*.bat text eol=crlf
*.sh text eol=lf
2 changes: 1 addition & 1 deletion .github/PULL_REQUEST_TEMPLATE.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ To upload images to a PR -- simply drag and drop an image while in edit mode and
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my feature works
- [ ] I have updated the changelog and the corresponding version in the extension's `config/extension.toml` file
- [ ] I have added a changelog fragment under `source/<pkg>/changelog.d/` for every touched package (do **not** edit `CHANGELOG.rst` or bump `extension.toml` — CI handles that)
- [ ] I have added my name to the `CONTRIBUTORS.md` or my name already exists there

<!--
Expand Down
44 changes: 19 additions & 25 deletions .github/actions/run-package-tests/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ inputs:
description: 'Additional pytest options'
default: ''
required: false
extra-pip-packages:
description: 'Space-separated pip packages to install inside the Docker container before pytest starts'
default: ''
required: false
container-name:
description: 'Docker container name prefix (run-id is appended automatically)'
required: true
Expand Down Expand Up @@ -83,6 +87,15 @@ runs:
echo "🟠 Could not obtain IMDS token, probably not running on EC2"
fi

- name: Prepare Docker disk space
shell: bash
run: |
bash .github/actions/run-package-tests/cleanup_docker_storage.sh \
"${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" \
"isaacsim-pin-${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}-pre" \
"pre-test-cleanup" \
"true"

- name: Record pull start time
id: pull-start
shell: bash
Expand Down Expand Up @@ -125,6 +138,7 @@ runs:
quarantined-only: ${{ inputs.quarantined-only }}
include-files: ${{ inputs.include-files }}
volume-mount-source: ${{ github.workspace }}
extra-pip-packages: ${{ inputs.extra-pip-packages }}

- name: Check Test Results
if: always()
Expand Down Expand Up @@ -153,28 +167,8 @@ runs:
if: always()
shell: bash
run: |
# Don't let cleanup errors fail a passing test job
set +e

echo "🔵 Cleaning up old Docker images..."

echo "⚪ Disk usage before cleanup:"
df -h /var/lib/docker

ISAACSIM_IMAGE="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}"

# Pin the base IsaacSim image so prune -a doesn't remove it
ISAACSIM_IMAGE_PIN_CONTAINER="isaacsim-pin-${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}"
echo "🔵 Pinning IsaacSim base image: ${ISAACSIM_IMAGE} to temporary container ${ISAACSIM_IMAGE_PIN_CONTAINER}"
docker create --name "${ISAACSIM_IMAGE_PIN_CONTAINER}" "${ISAACSIM_IMAGE}" true 2>/dev/null || true

echo "🔵 Removing Docker images older than 3 days..."
docker image prune -a -f --filter "until=72h"

echo "🔵 Removing temporary container ${ISAACSIM_IMAGE_PIN_CONTAINER}..."
docker rm "${ISAACSIM_IMAGE_PIN_CONTAINER}" 2>/dev/null || true

echo "🟢 Docker image cleanup complete."

echo "⚪ Disk usage after cleanup:"
df -h /var/lib/docker
bash .github/actions/run-package-tests/cleanup_docker_storage.sh \
"${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" \
"isaacsim-pin-${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}-post" \
"post-job-cleanup" \
"false"
128 changes: 128 additions & 0 deletions .github/actions/run-package-tests/cleanup_docker_storage.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
#!/usr/bin/env bash

# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause

set -u

ISAACSIM_IMAGE="${1:-}"
ISAACSIM_IMAGE_PIN_CONTAINER="${2:-isaacsim-pin-run-package-tests}"
CLEANUP_CONTEXT="${3:-docker-cleanup}"
FAIL_IF_STILL_HIGH="${4:-false}"

DOCKER_STORAGE_PATH="${DOCKER_STORAGE_PATH:-/var/lib/docker}"
DOCKER_CLEANUP_THRESHOLD_PERCENT="${DOCKER_CLEANUP_THRESHOLD_PERCENT:-85}"
DOCKER_AGGRESSIVE_PRUNE_UNTIL="${DOCKER_AGGRESSIVE_PRUNE_UNTIL:-24h}"
DOCKER_CONSERVATIVE_PRUNE_UNTIL="${DOCKER_CONSERVATIVE_PRUNE_UNTIL:-72h}"

get_docker_storage_usage_percent() {
df -P "${DOCKER_STORAGE_PATH}" 2>/dev/null | awk 'NR == 2 { gsub("%", "", $5); print $5 }'
}

resolve_docker_storage_path() {
local docker_root_dir
docker_root_dir="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
if [ -n "${docker_root_dir}" ]; then
DOCKER_STORAGE_PATH="${docker_root_dir}"
echo "[${CLEANUP_CONTEXT}] Using Docker root from docker info: ${DOCKER_STORAGE_PATH}"
return 0
fi

echo "[${CLEANUP_CONTEXT}] Could not determine Docker storage path from ${DOCKER_STORAGE_PATH} or docker info." >&2
return 1
}

print_docker_storage_usage() {
echo "[${CLEANUP_CONTEXT}] Docker storage usage:"
df -h "${DOCKER_STORAGE_PATH}" || true
docker system df || true
}

pin_isaacsim_image_if_present() {
if [ -z "${ISAACSIM_IMAGE}" ]; then
echo "[${CLEANUP_CONTEXT}] No IsaacSim base image provided; skipping image pin."
return
fi

if docker image inspect "${ISAACSIM_IMAGE}" >/dev/null 2>&1; then
echo "[${CLEANUP_CONTEXT}] Pinning IsaacSim base image: ${ISAACSIM_IMAGE}"
docker create --name "${ISAACSIM_IMAGE_PIN_CONTAINER}" "${ISAACSIM_IMAGE}" true >/dev/null 2>&1 || true
else
echo "[${CLEANUP_CONTEXT}] IsaacSim base image is not present locally; skipping image pin."
fi
}

remove_isaacsim_image_pin() {
docker rm "${ISAACSIM_IMAGE_PIN_CONTAINER}" >/dev/null 2>&1 || true
}

run_conservative_cleanup() {
echo "[${CLEANUP_CONTEXT}] Running conservative Docker cleanup."
echo "[${CLEANUP_CONTEXT}] Removing Docker images older than ${DOCKER_CONSERVATIVE_PRUNE_UNTIL}."
docker image prune -a -f --filter "until=${DOCKER_CONSERVATIVE_PRUNE_UNTIL}" || true
}

run_aggressive_cleanup() {
echo "[${CLEANUP_CONTEXT}] Running aggressive Docker cleanup."
echo "[${CLEANUP_CONTEXT}] Removing stopped Docker containers."
docker container prune -f --filter "until=${DOCKER_AGGRESSIVE_PRUNE_UNTIL}" || true

echo "[${CLEANUP_CONTEXT}] Removing unused Docker builder cache older than ${DOCKER_AGGRESSIVE_PRUNE_UNTIL}."
docker builder prune -a -f --filter "until=${DOCKER_AGGRESSIVE_PRUNE_UNTIL}" || true

echo "[${CLEANUP_CONTEXT}] Removing Docker images older than ${DOCKER_AGGRESSIVE_PRUNE_UNTIL}."
docker image prune -a -f --filter "until=${DOCKER_AGGRESSIVE_PRUNE_UNTIL}" || true
}

main() {
echo "[${CLEANUP_CONTEXT}] Checking Docker storage before cleanup."

local usage_percent
usage_percent="$(get_docker_storage_usage_percent)"
if [ -z "${usage_percent}" ]; then
resolve_docker_storage_path || true
usage_percent="$(get_docker_storage_usage_percent)"
fi

print_docker_storage_usage

if [ -z "${usage_percent}" ]; then
echo "[${CLEANUP_CONTEXT}] Could not determine Docker storage usage; running conservative cleanup."
pin_isaacsim_image_if_present
trap remove_isaacsim_image_pin EXIT
run_conservative_cleanup
print_docker_storage_usage
if [ "${FAIL_IF_STILL_HIGH}" = "true" ]; then
echo "::error::Could not determine Docker storage usage after cleanup. The self-hosted runner needs a measurable Docker storage path before tests can run." >&2
return 1
fi
return 0
fi

pin_isaacsim_image_if_present
trap remove_isaacsim_image_pin EXIT

if [ "${usage_percent}" -ge "${DOCKER_CLEANUP_THRESHOLD_PERCENT}" ]; then
echo "[${CLEANUP_CONTEXT}] Docker storage is at ${usage_percent}%, meeting the ${DOCKER_CLEANUP_THRESHOLD_PERCENT}% threshold."
run_aggressive_cleanup
else
echo "[${CLEANUP_CONTEXT}] Docker storage is at ${usage_percent}%, below the ${DOCKER_CLEANUP_THRESHOLD_PERCENT}% threshold."
run_conservative_cleanup
fi

echo "[${CLEANUP_CONTEXT}] Docker storage after cleanup."
print_docker_storage_usage

local final_usage_percent
final_usage_percent="$(get_docker_storage_usage_percent)"
if [ "${FAIL_IF_STILL_HIGH}" = "true" ] \
&& [ -n "${final_usage_percent}" ] \
&& [ "${final_usage_percent}" -ge "${DOCKER_CLEANUP_THRESHOLD_PERCENT}" ]; then
echo "::error::Docker storage remains at ${final_usage_percent}% after cleanup. The self-hosted runner needs more free Docker storage before tests can run." >&2
return 1
fi
}

main "$@"
181 changes: 181 additions & 0 deletions .github/actions/run-package-tests/test_cleanup_docker_storage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause

"""Tests for CI Docker storage cleanup helpers."""

from __future__ import annotations

import os
import subprocess
from pathlib import Path

_ACTION_DIR = Path(__file__).resolve().parent
_REPO_ROOT = _ACTION_DIR.parents[2]
_ACTION_PATH = _ACTION_DIR / "action.yml"
_SCRIPT_PATH = _ACTION_DIR / "cleanup_docker_storage.sh"


def _write_fake_df(
bin_dir: Path, first_usage: int, final_usage: int, failed_path: str | None = None, fallback_path: str | None = None
) -> None:
count_file = bin_dir / "df-count"
count_file.write_text("0", encoding="utf-8")
script = bin_dir / "df"
script.write_text(
f"""#!/usr/bin/env bash
set -euo pipefail

path="${{2:-}}"
if [ -n "{failed_path or ""}" ] && [ "$path" = "{failed_path or ""}" ]; then
exit 1
fi
if [ -n "{fallback_path or ""}" ] && [ "$path" = "{fallback_path or ""}" ]; then
:
elif [ -n "{fallback_path or ""}" ]; then
exit 1
fi

if [ "${{1:-}}" = "-P" ]; then
count=$(cat "{count_file}")
if [ "$count" -eq 0 ]; then
usage="{first_usage}"
else
usage="{final_usage}"
fi
echo "$((count + 1))" > "{count_file}"
echo "Filesystem 1024-blocks Used Available Capacity Mounted on"
echo "/dev/test 100 90 10 ${{usage}}% /var/lib/docker"
else
echo "Filesystem Size Used Avail Use% Mounted on"
echo "/dev/test 100G 90G 10G {first_usage}% /var/lib/docker"
fi
""",
encoding="utf-8",
)
script.chmod(0o755)


def _write_fake_docker(bin_dir: Path, docker_root_dir: str = "/var/lib/docker") -> Path:
log_path = bin_dir / "docker.log"
script = bin_dir / "docker"
script.write_text(
f"""#!/usr/bin/env bash
set -euo pipefail

printf '%s\\n' "$*" >> "{log_path}"
if [ "${{1:-}}" = "info" ] && [ "${{2:-}}" = "--format" ]; then
if [ -z "{docker_root_dir}" ]; then
exit 1
fi
echo "{docker_root_dir}"
exit 0
fi
if [ "${{1:-}}" = "image" ] && [ "${{2:-}}" = "inspect" ]; then
exit 0
fi
exit 0
""",
encoding="utf-8",
)
script.chmod(0o755)
return log_path


def _run_cleanup(
tmp_path: Path,
first_usage: int,
final_usage: int,
failed_path: str | None = None,
docker_root_dir: str = "/var/lib/docker",
) -> subprocess.CompletedProcess[str]:
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
_write_fake_df(bin_dir, first_usage, final_usage, failed_path=failed_path, fallback_path=docker_root_dir)
_write_fake_docker(bin_dir, docker_root_dir=docker_root_dir)

env = os.environ.copy()
env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
env["DOCKER_STORAGE_PATH"] = "/var/lib/docker"
env["DOCKER_CLEANUP_THRESHOLD_PERCENT"] = "85"

return subprocess.run(
[
"bash",
str(_SCRIPT_PATH),
"nvcr.io/nvidian/isaac-sim:latest-develop",
"isaacsim-pin-test",
"test-cleanup",
"true",
],
cwd=_REPO_ROOT,
env=env,
check=False,
capture_output=True,
text=True,
)


def test_cleanup_uses_aggressive_pruning_when_docker_storage_exceeds_threshold(tmp_path: Path):
result = _run_cleanup(tmp_path, first_usage=91, final_usage=70)

assert result.returncode == 0, result.stderr
docker_log = (tmp_path / "bin" / "docker.log").read_text(encoding="utf-8")
assert "container prune -f --filter until=24h" in docker_log
assert "builder prune -a -f --filter until=24h" in docker_log
assert "image prune -a -f --filter until=24h" in docker_log


def test_cleanup_keeps_conservative_image_pruning_below_threshold(tmp_path: Path):
result = _run_cleanup(tmp_path, first_usage=70, final_usage=70)

assert result.returncode == 0, result.stderr
docker_log = (tmp_path / "bin" / "docker.log").read_text(encoding="utf-8")
assert "image prune -a -f --filter until=72h" in docker_log
assert "builder prune" not in docker_log


def test_cleanup_fails_early_when_aggressive_pruning_cannot_recover_space(tmp_path: Path):
result = _run_cleanup(tmp_path, first_usage=91, final_usage=89)

assert result.returncode == 1
assert "Docker storage remains at 89%" in result.stderr


def test_cleanup_uses_docker_root_dir_when_default_storage_path_is_unavailable(tmp_path: Path):
result = _run_cleanup(
tmp_path,
first_usage=91,
final_usage=89,
failed_path="/var/lib/docker",
docker_root_dir="/mnt/docker-root",
)

assert result.returncode == 1
assert "Using Docker root from docker info: /mnt/docker-root" in result.stdout
assert "Docker storage remains at 89%" in result.stderr


def test_cleanup_fails_early_when_docker_storage_usage_cannot_be_measured(tmp_path: Path):
result = _run_cleanup(
tmp_path,
first_usage=91,
final_usage=89,
failed_path="/var/lib/docker",
docker_root_dir="",
)

assert result.returncode == 1
assert "Could not determine Docker storage usage after cleanup" in result.stderr


def test_run_package_tests_prepares_docker_space_before_image_pull():
action_text = _ACTION_PATH.read_text(encoding="utf-8")

cleanup_index = action_text.index("name: Prepare Docker disk space")
pull_start_index = action_text.index("name: Record pull start time")
pull_index = action_text.index("name: Pull image from ECR")

assert cleanup_index < pull_start_index < pull_index
assert "cleanup_docker_storage.sh" in action_text
Loading
Loading