diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 0cad2933e949..841d395afa32 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -8,12 +8,6 @@ name: Build PIP Wheel on: pull_request: types: [opened, synchronize, reopened] - paths: - - 'apps/**' - - 'VERSION' - - 'source/**' - - 'tools/wheel_builder/**' - - '.github/workflows/wheel.yml' push: branches: - main @@ -26,6 +20,7 @@ concurrency: permissions: contents: read + pull-requests: read jobs: build-wheel: @@ -34,13 +29,103 @@ jobs: timeout-minutes: 5 steps: + - name: Detect wheel-relevant changes + id: changes + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_NAME: ${{ github.event_name }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + + # Keep this workflow unconditionally triggered on PRs so required + # branch-protection checks are always reported. The build steps below + # run only when inputs that can affect the wheel have changed. + patterns=( + $'^apps/\tStandalone apps packaged in the wheel' + $'^VERSION$\tPackage version' + $'^source/\tPython packages' + $'^tools/wheel_builder/\tWheel build tooling' + $'^\.github/workflows/wheel\.yml$\tThis workflow file' + ) + + render_table() { + local files="$1" entry regex desc count sample + echo "| Pattern | What it covers | Matched files |" + echo "|---|---|---|" + for entry in "${patterns[@]}"; do + IFS=$'\t' read -r regex desc <<< "$entry" + count=$(printf '%s\n' "$files" | grep -cE "$regex" || true) + if [ "$count" -gt 0 ]; then + sample=$(printf '%s\n' "$files" | grep -E "$regex" | head -3 | paste -sd ', ' -) + [ "$count" -gt 3 ] && sample="$sample (and $((count - 3)) more)" + echo "| \`$regex\` | $desc | $sample |" + else + echo "| \`$regex\` | $desc | - |" + fi + done + } + + any_match() { + local files="$1" entry regex + for entry in "${patterns[@]}"; do + IFS=$'\t' read -r regex _ <<< "$entry" + if printf '%s\n' "$files" | grep -qE "$regex"; then + return 0 + fi + done + return 1 + } + + decide() { + local decision="$1" reason="$2" files="${3:-}" + echo "run_build=$decision" >> "$GITHUB_OUTPUT" + { + echo "## Wheel build gating" + echo "" + if [ "$decision" = "true" ]; then + echo "The wheel build will **run**: $reason." + else + echo "The wheel build will be **skipped**: $reason." + fi + if [ -n "$files" ]; then + echo "" + render_table "$files" + fi + } >> "$GITHUB_STEP_SUMMARY" + } + + if [ "$EVENT_NAME" != "pull_request" ]; then + decide true "non-PR event ($EVENT_NAME)" + exit 0 + fi + + if ! changed_files="$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/files" --jq '.[].filename')"; then + echo "::warning::Could not list changed files; defaulting to building the wheel" + decide true "fail-safe (could not list changed files)" + exit 0 + fi + + if any_match "$changed_files"; then + decide true "wheel-relevant paths changed" "$changed_files" + else + decide false "no wheel-relevant paths changed" "$changed_files" + fi + + - name: Skip wheel build + if: steps.changes.outputs.run_build == 'false' + run: echo "Skipping wheel build because this PR does not change wheel inputs." + - name: Checkout code + if: steps.changes.outputs.run_build == 'true' uses: actions/checkout@v6 with: fetch-depth: 1 lfs: true - name: Setup Python + if: steps.changes.outputs.run_build == 'true' uses: actions/setup-python@v5 with: python-version: "3.12" @@ -51,6 +136,7 @@ jobs: # isaaclab--build-. The wheel inside follows # PEP 440 (VERSION+buildN.SHA7) since pip requires that format. - name: Compute wheel metadata + if: steps.changes.outputs.run_build == 'true' id: meta run: | set -euo pipefail @@ -59,12 +145,14 @@ jobs: echo "artifact_name=isaaclab-${version}-build${{ github.run_number }}-${sha_slug}" >> "$GITHUB_OUTPUT" - name: Build wheel + if: steps.changes.outputs.run_build == 'true' env: WHEEL_BUILD_NUMBER: ${{ github.run_number }} WHEEL_SHA: ${{ github.sha }} run: bash tools/wheel_builder/build.sh - name: Upload wheel artifact + if: steps.changes.outputs.run_build == 'true' uses: actions/upload-artifact@v7 with: name: ${{ steps.meta.outputs.artifact_name }} diff --git a/docs/index.rst b/docs/index.rst index 6c8e6ba9ac41..d3218514afb7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -143,6 +143,7 @@ Table of Contents source/experimental-features/bleeding-edge source/experimental-features/newton-physics-integration/index + source/experimental-features/visuo_tactile_sensor .. toctree:: :maxdepth: 1 diff --git a/docs/source/_static/setup/ecosystem-dark.jpg b/docs/source/_static/setup/ecosystem-dark.jpg deleted file mode 100644 index 9057aeb48b29..000000000000 Binary files a/docs/source/_static/setup/ecosystem-dark.jpg and /dev/null differ diff --git a/docs/source/_static/setup/ecosystem-dark.svg b/docs/source/_static/setup/ecosystem-dark.svg new file mode 100644 index 000000000000..a09f1d0c5036 --- /dev/null +++ b/docs/source/_static/setup/ecosystem-dark.svg @@ -0,0 +1,93 @@ + + + + + + + + + + + + Learning Libraries + RSL-RL · skrl · Stable Baselines 3 · RL Games + + + + Isaac Lab Extensions + + isaaclab_tasks + pre-built environments + + isaaclab_assets + robot & sensor configs + + isaaclab_rl + RL library wrappers + + isaaclab_mimic + imitation learning + + isaaclab_teleop + teleoperation & XR + + + + Isaac Lab Core (isaaclab) + sim · scene · assets · sensors · envs · managers · actuators + controllers · terrains · devices · mdp · utils · markers · renderers + ManagerBasedRLEnv · DirectRLEnv · DirectMARLEnv · InteractiveScene + factory pattern: uniform API dispatches to the active physics backend at runtime + + + + + + isaaclab_physx · isaaclab_ovphysx · isaaclab_ov + Articulation · Rigid Body · Rigid Object Collection · Deformable + Fabric Views · Camera · USD spawners + Isaac RTX Renderer · OvRTX Renderer + + + + isaaclab_newton + Articulation · Rigid Body · Rigid Object Collection + Camera · USD spawners · Warp Renderer + MJWarp Solver · VBD Solver + + + + + + Isaac Sim (optional) + PhysX · RTX Rendering · USD / Omniverse + ROS / ROS 2 · URDF & MJCF importers + required for Isaac Sim features + + + + Newton Physics + Warp + GPU-parallel simulation · Warp kernels + MJWarp (MuJoCo-Warp) solver + kit-less · no Isaac Sim required + + + + NVIDIA GPU Platform + CUDA · Warp (required by all backends) + diff --git a/docs/source/_static/setup/ecosystem-light.jpg b/docs/source/_static/setup/ecosystem-light.jpg deleted file mode 100644 index 5c2bbf5fe151..000000000000 Binary files a/docs/source/_static/setup/ecosystem-light.jpg and /dev/null differ diff --git a/docs/source/_static/setup/ecosystem-light.svg b/docs/source/_static/setup/ecosystem-light.svg new file mode 100644 index 000000000000..cd9b1b98fb37 --- /dev/null +++ b/docs/source/_static/setup/ecosystem-light.svg @@ -0,0 +1,94 @@ + + + + + + + + + + + + Learning Libraries + RSL-RL · skrl · Stable Baselines 3 · RL Games + + + + Isaac Lab Extensions + + isaaclab_tasks + pre-built environments + + isaaclab_assets + robot & sensor configs + + isaaclab_rl + RL library wrappers + + isaaclab_mimic + imitation learning + + isaaclab_teleop + teleoperation & XR + + + + Isaac Lab Core (isaaclab) + sim · scene · assets · sensors · envs · managers · actuators + controllers · terrains · devices · mdp · utils · markers · renderers + ManagerBasedRLEnv · DirectRLEnv · DirectMARLEnv · InteractiveScene + factory pattern: uniform API dispatches to the active physics backend at runtime + + + + + + isaaclab_physx · isaaclab_ovphysx · isaaclab_ov + Articulation · Rigid Body · Rigid Object Collection · Deformable + Fabric Views · Camera · USD spawners + Isaac RTX Renderer · OvRTX Renderer + + + + isaaclab_newton + Articulation · Rigid Body · Rigid Object Collection + Camera · USD spawners · Warp Renderer + MJWarp Solver · VBD Solver + + + + + + Isaac Sim (optional) + PhysX · RTX Rendering · USD / Omniverse + ROS / ROS 2 · URDF & MJCF importers + required for Isaac Sim features + + + + Newton Physics + Warp + GPU-parallel simulation · Warp kernels + MJWarp (MuJoCo-Warp) solver + kit-less · no Isaac Sim required + + + + NVIDIA GPU Platform + CUDA · Warp (required by all backends) + diff --git a/docs/source/overview/core-concepts/sensors/visuo_tactile_sensor.rst b/docs/source/experimental-features/visuo_tactile_sensor.rst similarity index 97% rename from docs/source/overview/core-concepts/sensors/visuo_tactile_sensor.rst rename to docs/source/experimental-features/visuo_tactile_sensor.rst index e1b389e45c1c..4e3d3897c6ab 100644 --- a/docs/source/overview/core-concepts/sensors/visuo_tactile_sensor.rst +++ b/docs/source/experimental-features/visuo_tactile_sensor.rst @@ -9,7 +9,7 @@ Visuo-Tactile Sensor The visuo-tactile sensor in Isaac Lab provides realistic tactile feedback through integration with TacSL (Tactile Sensor Learning) [Akinola2025]_. It is designed to simulate high-fidelity tactile interactions, generating both visual and force-based data that mirror real-world tactile sensors like GelSight devices. The sensor can provide tactile RGB images, force field distributions, and other intermediate tactile measurements essential for robotic manipulation tasks requiring fine tactile feedback. -.. figure:: ../../../_static/overview/sensors/tacsl_diagram.jpg +.. figure:: ../_static/overview/sensors/tacsl_diagram.jpg :align: center :figwidth: 100% :alt: Tactile sensor with RGB visualization and force fields @@ -142,7 +142,7 @@ For a complete list of available options: .. note:: The demo examples are based on the Gelsight R1.5, which is a prototype sensor that is now discontinued. The same procedure can be adapted for other visuotactile sensors. -.. figure:: ../../../_static/overview/sensors/tacsl_demo.jpg +.. figure:: ../_static/overview/sensors/tacsl_demo.jpg :align: center :figwidth: 100% :alt: TacSL tactile sensor demo showing RGB tactile images and force field visualizations @@ -163,12 +163,12 @@ Output Tactile Data :widths: 50 50 :class: borderless - * - .. figure:: ../../../_static/overview/sensors/tacsl_taxim_example.jpg + * - .. figure:: ../_static/overview/sensors/tacsl_taxim_example.jpg :align: center :figwidth: 80% :alt: Tactile output with RGB visualization - - .. figure:: ../../../_static/overview/sensors/tacsl_force_field_example.jpg + - .. figure:: ../_static/overview/sensors/tacsl_force_field_example.jpg :align: center :figwidth: 80% :alt: Tactile output with force field visualization diff --git a/docs/source/overview/core-concepts/sensors/index.rst b/docs/source/overview/core-concepts/sensors/index.rst index f32923ab9c30..1ecae8e3ccdf 100644 --- a/docs/source/overview/core-concepts/sensors/index.rst +++ b/docs/source/overview/core-concepts/sensors/index.rst @@ -21,4 +21,3 @@ The following pages describe the available sensors in more detail: pva joint_wrench_sensor ray_caster - visuo_tactile_sensor diff --git a/docs/source/overview/developer-guide/repo_structure.rst b/docs/source/overview/developer-guide/repo_structure.rst index d8a5a1b200a5..92fd0f90dcdd 100644 --- a/docs/source/overview/developer-guide/repo_structure.rst +++ b/docs/source/overview/developer-guide/repo_structure.rst @@ -15,56 +15,113 @@ Repository organization ├── docs ├── docker ├── source - │   ├── isaaclab - │   ├── isaaclab_assets - │   ├── isaaclab_mimic - │   ├── isaaclab_rl - │   └── isaaclab_tasks + │ ├── isaaclab # core framework + │ ├── isaaclab_physx # PhysX backend (requires Isaac Sim) + │ ├── isaaclab_ovphysx # standalone PhysX backend (requires ovphysx) + │ ├── isaaclab_ov # OVRTX renderer backend (requires ovrtx) + │ ├── isaaclab_newton # Newton backend (kit-less) + │ ├── isaaclab_assets # pre-configured robot & sensor assets + │ ├── isaaclab_tasks # pre-built RL/IL environments + │ ├── isaaclab_tasks_experimental # Warp-accelerated environments + │ ├── isaaclab_rl # RL library wrappers + │ ├── isaaclab_mimic # imitation learning & data generation + │ ├── isaaclab_teleop # teleoperation & XR + │ ├── isaaclab_visualizers # external visualizer backends + │ ├── isaaclab_contrib # community-contributed extensions + │ ├── isaaclab_experimental # Warp-accelerated manager and environment variants + │ ├── extensions # legacy Omniverse extension wrappers + │ └── standalone # standalone tutorials & workflows ├── scripts - │   ├── benchmarks - │   ├── demos - │   ├── environments - │   ├── imitation_learning - │   ├── reinforcement_learning - │   ├── tools - │   ├── tutorials + │ ├── benchmarks + │ ├── demos + │ ├── environments + │ ├── imitation_learning + │ ├── reinforcement_learning + │ ├── sim2sim_transfer + │ ├── tools + │ └── tutorials ├── tools └── VERSION -Isaac Lab is built on the same back end as Isaac Sim. As such, it exists as a collection of **extensions** that can be assembled into **applications**. -The ``source`` directory contains the majority of the code in the repository and the specific extensions that compose Isaac lab, while ``scripts`` containing python scripts for launching customized standalone apps (Like our workflows). -These are the two primary ways of interacting with the simulation and Isaac lab supports both! -Checkout this `Isaac Sim introduction to workflows `__ for more details. +Isaac Lab supports the **PhysX** and **Newton** physics engines through backend packages. The +default PhysX path runs through Isaac Sim, while ``ovphysx`` supports standalone PhysX workflows +without launching Isaac Sim and Newton provides a Warp-native kit-less backend. The ``source`` +directory contains all packages that compose Isaac Lab, while ``scripts`` contains standalone +Python applications for training, evaluation, and tooling. +See :doc:`/source/overview/core-concepts/multi_backend_architecture` for details on the backend +system, and :doc:`/source/setup/ecosystem` for a full package-layer overview. -Extensions +Submodules ~~~~~~~~~~ -The extensions that compose Isaac Lab are kept in the ``source`` directory. To simplify the build process, Isaac Lab directly use `setuptools `__. It is strongly recommend that you adhere to this process if you create your own extensions using Isaac Lab. +The packages under ``source/`` are installed as Python packages using +`setuptools `__. They are organized into three +groups: -The extensions are organized as follows: +**Core, physics backends, and renderers** -* **isaaclab**: Contains the core interface extension for Isaac Lab. This provides the main modules for actuators, - objects, robots and sensors. -* **isaaclab_assets**: Contains the extension with pre-configured assets for Isaac Lab. -* **isaaclab_tasks**: Contains the extension with pre-configured environments for Isaac Lab. -* **isaaclab_mimic**: Contains APIs and pre-configured environments for data generation for imitation learning. -* **isaaclab_rl**: Contains wrappers for using the above environments with different reinforcement learning agents. +* **isaaclab**: The core framework. Provides :mod:`~isaaclab.sim` (simulation context and + configuration), :class:`~isaaclab.scene.InteractiveScene`, asset and sensor base classes and + factory interfaces (:mod:`~isaaclab.assets`, :mod:`~isaaclab.sensors`), environment base + classes (:mod:`~isaaclab.envs`), the manager system (:mod:`~isaaclab.managers`), composable + MDP term library (:mod:`~isaaclab.envs.mdp`), actuator models (:mod:`~isaaclab.actuators`), + low-level controllers (:mod:`~isaaclab.controllers`), procedural terrain generation + (:mod:`~isaaclab.terrains`), and human-input device support (:mod:`~isaaclab.devices`). +* **isaaclab_physx**: PhysX-backed implementations of articulations, rigid bodies, deformable + objects, Fabric views, the Isaac RTX renderer, and USD spawners. Requires Isaac Sim. +* **isaaclab_ovphysx**: Standalone PhysX backend variant using ``ovphysx`` and the + TensorBindingsAPI. Requires the ``ovphysx`` package and can run without launching Isaac Sim. +* **isaaclab_ov**: OVRTX renderer backend for RTX-based tiled camera rendering. Requires the + ``ovrtx`` package and can run without Isaac Sim. +* **isaaclab_newton**: Newton-backed implementations of articulations, rigid bodies, rigid + object collections, cameras, USD spawners, and the Warp renderer. Supports + :ref:`kit-less installation ` without Isaac Sim. + +**Tasks and assets** + +* **isaaclab_assets**: Pre-configured :class:`~isaaclab.utils.configclass` dataclasses for a + wide range of robots (Franka, Unitree, ANYmal, Spot, Allegro, humanoids, quadcopters, and + more) and sensors (Velodyne, GelSight). +* **isaaclab_tasks**: Registered `gymnasium `__ environments for + reinforcement and imitation learning, organized as *manager-based* and *direct* tasks across + locomotion, manipulation, navigation, and classic control domains. +* **isaaclab_tasks_experimental**: Experimental task implementations under active development, + not yet part of the stable task suite. + +**Optional Additions** + +* **isaaclab_rl**: Thin wrappers that adapt Isaac Lab environments to the interfaces expected + by `RSL-RL `__, + `skrl `__, + `Stable Baselines 3 `__, and + `RL Games `__. +* **isaaclab_mimic**: APIs and pre-configured environments for data generation and imitation + learning, including cuRobo-based motion planners and a dataset-generation pipeline. +* **isaaclab_teleop**: Teleoperation session orchestration with OpenXR / CloudXR support, + device retargeters for manipulators and humanoids, and gamepad / spacemouse / keyboard input. +* **isaaclab_visualizers**: Supplementary visualizer backends (Isaac Sim Kit, Newton, Rerun, Viser) that + work with any physics backend. +* **isaaclab_contrib**: Community-contributed features: multirotor assets, TacSL + visuo-tactile sensors, drone thrust controllers, and more. +* **isaaclab_experimental**: Pre-production core experiments including Warp-accelerated manager + and environment variants. Standalone ~~~~~~~~~~ -The ``scripts`` directory contains various standalone applications written in python. +The ``scripts`` directory contains standalone Python applications. They are structured as follows: -* **benchmarks**: Contains scripts for benchmarking different framework components. -* **demos**: Contains various demo applications that showcase the core framework :mod:`isaaclab`. -* **environments**: Contains applications for running environments defined in :mod:`isaaclab_tasks` with - different agents. These include a random policy, zero-action policy, teleoperation or scripted state machines. -* **imitation_learning**: Contains applications for training and evaluating policies with various - imitation learning libraries (e.g. robomimic). -* **reinforcement_learning**: Contains applications for training and evaluating policies with various - reinforcement learning libraries (e.g. rsl_rl, rl_games, sb3, skrl). -* **tools**: Contains applications for using the tools provided by the framework. These include converting assets, - generating datasets, etc. -* **tutorials**: Contains step-by-step tutorials for using the APIs provided by the framework. +* **benchmarks**: Scripts for benchmarking different framework components. +* **demos**: Demo applications that showcase the core framework :mod:`isaaclab`. +* **environments**: Scripts for running environments defined in :mod:`isaaclab_tasks` with + different agents (random policy, zero-action policy, teleoperation, scripted state machines). +* **imitation_learning**: Applications for training and evaluating policies with imitation + learning libraries (e.g. robomimic). +* **reinforcement_learning**: Applications for training and evaluating policies with RL + libraries (e.g. rsl_rl, rl_games, sb3, skrl). +* **sim2sim_transfer**: Scripts for transferring policies trained in one simulator to another. +* **tools**: Applications for using framework tools such as converting assets and generating + datasets. +* **tutorials**: Step-by-step tutorials for using the APIs provided by the framework. diff --git a/docs/source/setup/ecosystem.rst b/docs/source/setup/ecosystem.rst index 75c22b24313e..17105bbea5cd 100644 --- a/docs/source/setup/ecosystem.rst +++ b/docs/source/setup/ecosystem.rst @@ -3,156 +3,209 @@ Isaac Lab Ecosystem =================== -Isaac Lab is built on top of Isaac Sim and Newton to provide a unified and flexible framework -for robot learning that exploits the latest simulation technologies. It is designed to be modular and extensible, -and aims to simplify common workflows in robotics research (such as RL, learning from demonstrations, and -motion planning). While it includes some pre-built environments, sensors, and tasks, its main goal is to -provide an open-sourced, unified, and easy-to-use interface for developing and testing custom environments -and robot learning algorithms. - -Working with Isaac Lab requires the installation of Isaac Sim for full functionality, which is packaged with -core robotics tools including URDF and MJCF importers, and ROS features. Isaac Sim also builds on top of the NVIDIA -Omniverse platform, leveraging advanced physics simulation from **PhysX**, photorealistic **RTX** rendering -technologies, and Universal Scene Description (USD) for scene creation. Without Isaac Sim, users can still use -the **Newton** physics backend as well as the new **OVRTX** renderers. +Isaac Lab is a modular, extensible framework for robot learning built on top of `Isaac Sim`_ and +`Newton`_. It provides a unified interface for the most common workflows in robotics research — +reinforcement learning, learning from demonstrations, and motion planning — while staying easy to +use and easy to extend. + +Isaac Lab supports two physics engines through multiple backend packages: + +* **PhysX** — the default backend through `Isaac Sim`_, with access to GPU-accelerated + rigid-body simulation, deformable objects, Fabric views, tiled RTX rendering, ROS/ROS2, + URDF/MJCF importers, and the full Omniverse toolchain. PhysX can also be used through the + standalone ``ovphysx`` runtime for kit-less workflows that do not launch Isaac Sim. +* **Newton** — a Warp-native backend that can run in kit-less mode, enabling lightweight + deployments and GPU-parallel simulation using `Warp`_. .. note:: - Isaac Lab 3.0 supports a **kit-less installation** mode: you can install Isaac Lab and use the Newton - physics backend without installing Isaac Sim at all. See :ref:`isaaclab-installation-root` for details. + Isaac Lab 3.0 supports a **kit-less installation** mode: you can install Isaac Lab and use the + Newton physics backend without installing Isaac Sim at all. + See :ref:`isaaclab-installation-root` for details. -Isaac Lab not only inherits the capabilities of Isaac Sim, but also adds a number -of new features that pertain to robot learning research. For example, including actuator dynamics in the -simulation, procedural terrain generation, and support to collect data from human demonstrations. +A factory pattern dispatches every asset and sensor instantiation to the correct backend at +runtime, so user code stays unchanged regardless of which backend is active. See +:doc:`/source/overview/core-concepts/multi_backend_architecture` for details. -.. image:: ../_static/setup/ecosystem-light.jpg +.. image:: ../_static/setup/ecosystem-light.svg :class: only-light :align: center - :alt: The Isaac Lab, Isaac Sim, and NVIDIA Omniverse ecosystem + :alt: The Isaac Lab package ecosystem layered on the NVIDIA GPU platform -.. image:: ../_static/setup/ecosystem-dark.jpg +.. image:: ../_static/setup/ecosystem-dark.svg :class: only-dark :align: center - :alt: The Isaac Lab, Isaac Sim, and NVIDIA Omniverse ecosystem + :alt: The Isaac Lab package ecosystem layered on the NVIDIA GPU platform + + +Package structure +----------------- + +Isaac Lab is organized into a set of focused packages that can be used independently or together. + +**Core** + +* ``isaaclab`` — the core library. Contains simulation context and configuration + (:mod:`~isaaclab.sim`), the :class:`~isaaclab.scene.InteractiveScene` that aggregates all + assets, sensors, and terrain for a vectorized set of environments, asset interfaces + (:mod:`~isaaclab.assets`), sensor interfaces (:mod:`~isaaclab.sensors`), environment base + classes (:mod:`~isaaclab.envs`), the manager system (:mod:`~isaaclab.managers`), + composable MDP term functions (:mod:`~isaaclab.envs.mdp`), actuator models + (:mod:`~isaaclab.actuators`), low-level controllers (:mod:`~isaaclab.controllers`), + procedural terrain generation (:mod:`~isaaclab.terrains`), and human-input device support + (:mod:`~isaaclab.devices`). + +**Physics and renderer backends** + +* ``isaaclab_physx`` — PhysX-backed implementations of articulations, + rigid bodies, deformable objects, Fabric views, the Isaac RTX renderer, and USD spawners. + Requires Isaac Sim. +* ``isaaclab_ovphysx`` — standalone PhysX-backed implementations built on ``ovphysx`` and + TensorBindingsAPI. Requires the ``ovphysx`` package and can run without launching Isaac Sim. +* ``isaaclab_ov`` — Omniverse renderer package that provides the OVRTX renderer for + RTX-based tiled camera rendering. Requires the ``ovrtx`` package and can be used in + kit-less workflows without Isaac Sim. +* ``isaaclab_newton`` — Newton-backed implementations of articulations, rigid bodies, and the + Warp renderer. Supports kit-less installation without Isaac Sim. + +**Extensions** + +* ``isaaclab_assets`` — pre-configured robot and sensor :class:`~isaaclab.utils.configclass` + dataclasses for a wide range of robots (Franka, Unitree, ANYmal, Spot, Allegro, humanoids, + quadcopters, and more) and sensors (Velodyne, GelSight). +* ``isaaclab_tasks`` — registered `gymnasium`_ environments organized into two authoring patterns: + + * *Manager-based* — behavior is fully specified through composable manager configurations + (observations, rewards, terminations, events, commands, actions). Well-suited for research + that requires clean separation between task specification and environment logic. + * *Direct* — a single Python class implements the full step/reset/obs/reward loop, similar + in style to Isaac Gym. Convenient for rapid prototyping and tasks with complex custom logic. + +* ``isaaclab_rl`` — thin wrappers that adapt Isaac Lab environments to the vectorized + environment interfaces expected by `RSL-RL`_, `skrl`_, `Stable Baselines 3`_, and + `RL Games`_. +* ``isaaclab_mimic`` — APIs and pre-configured environments for data generation and imitation + learning, including cuRobo-based motion planners and a full dataset-generation pipeline. +* ``isaaclab_teleop`` — teleoperation session orchestration with XR (OpenXR / CloudXR) support, + device retargeters for manipulators and humanoids, and gamepad/spacemouse/keyboard input. +* ``isaaclab_visualizers`` — supplementary visualizer backends (Isaac Kit, Rerun, Viser) that + work with any physics backend. +* ``isaaclab_contrib`` — community-contributed features: multirotor assets, TacSL visuo-tactile + sensors, drone thrust controllers, and more. +* ``isaaclab_experimental`` — pre-production experiments, including Warp-accelerated manager and + environment variants. Where does Isaac Lab fit in the Isaac ecosystem? ------------------------------------------------ Over the years, NVIDIA has developed a number of tools for robotics and AI. These tools leverage -the power of GPUs to accelerate the simulation both in terms of speed and realism. They show great -promise in the field of simulation technology and are being used by many researchers and companies -worldwide. - -`Isaac Gym`_ :cite:`makoviychuk2021isaac` provides a high performance GPU-based physics simulation -for robot learning. It is built on top of `PhysX`_ which supports GPU-accelerated simulation of rigid bodies -and a Python API to directly access physics simulation data. Isaac Lab extends this foundation with -additional support for the **Newton** physics backend, enabling broader simulation options. Through an end-to-end GPU pipeline, it is possible -to achieve high frame rates compared to CPU-based physics engines. The tool has been used successfully in a -number of research projects, including legged locomotion :cite:`rudin2022learning` :cite:`rudin2022advanced`, -in-hand manipulation :cite:`handa2022dextreme` :cite:`allshire2022transferring`, and industrial assembly -:cite:`narang2022factory`. - -Despite the success of Isaac Gym, it is not designed to be a general purpose simulator for -robotics. For example, it does not include interaction between deformable and rigid objects, high-fidelity -rendering, and support for ROS. The tool has been primarily designed as a preview release to showcase the -capabilities of the underlying physics engine. With the release of `Isaac Sim`_, NVIDIA is building -a general purpose simulator for robotics and has integrated the functionalities of Isaac Gym into -Isaac Sim. - -`Isaac Sim`_ is a robot simulation toolkit built on top of Omniverse, which is a general purpose platform -that aims to unite complex 3D workflows. Isaac Sim leverages the latest advances in graphics and -physics simulation to provide a high-fidelity simulation environment for robotics. It supports -ROS/ROS2, various sensor simulation, tools for domain randomization and synthetic data creation. -Tiled rendering support in Isaac Sim allows for vectorized rendering across environments, along with -support for running in the cloud using `Isaac Automator`_. -Overall, it is a powerful tool for roboticists and is a huge step forward in the field of robotics -simulation. - -With the release of above two tools, NVIDIA also released an open-sourced set of environments called -`IsaacGymEnvs`_ and `OmniIsaacGymEnvs`_, that have been built on top of Isaac Gym and Isaac Sim respectively. -These environments have been designed to display the capabilities of the underlying simulators and provide -a starting point to understand what is possible with the simulators for robot learning. These environments -can be used for benchmarking but are not designed for developing and testing custom environments and algorithms. -This is where Isaac Lab comes in. - -Isaac Lab is built on top of Isaac Sim to provide a unified and flexible framework -for robot learning that exploits latest simulation technologies. It is designed to be modular and extensible, -and aims to simplify common workflows in robotics research (such as RL, learning from demonstrations, and -motion planning). While it includes some pre-built environments, sensors, and tasks, its main goal is to -provide an open-sourced, unified, and easy-to-use interface for developing and testing custom environments -and robot learning algorithms. It not only inherits the capabilities of Isaac Sim, but also adds a number -of new features that pertain to robot learning research. For example, including actuator dynamics in the -simulation, procedural terrain generation, and support to collect data from human demonstrations. - -Isaac Lab replaces the previous `IsaacGymEnvs`_, `OmniIsaacGymEnvs`_ and `Orbit`_ frameworks and will -be the single robot learning framework for Isaac Sim. Previously released frameworks are deprecated -and we encourage users to follow our migration guides to transition over to Isaac Lab. +the power of GPUs to accelerate simulation both in terms of speed and realism. + +`Isaac Gym`_ :cite:`makoviychuk2021isaac` provided a high-performance GPU-based physics simulation +for robot learning built on top of `PhysX`_. Its end-to-end GPU pipeline enabled frame rates +far beyond what CPU-based physics engines could achieve. The tool proved successful across a +number of research projects, including legged locomotion :cite:`rudin2022learning` +:cite:`rudin2022advanced`, in-hand manipulation :cite:`handa2022dextreme` +:cite:`allshire2022transferring`, and industrial assembly :cite:`narang2022factory`. + +`Isaac Sim`_ is a general-purpose robot simulation toolkit built on top of `Omniverse`_. It +integrates the capabilities of Isaac Gym while adding high-fidelity rendering, ROS/ROS2, +deformable-object simulation, synthetic data generation, domain randomization, tiled rendering +for vectorized observations, and cloud support via `Isaac Automator`_. With the Isaac Gym legacy +API absorbed into Isaac Sim, NVIDIA also released open-sourced environment collections +`IsaacGymEnvs`_ and `OmniIsaacGymEnvs`_ to showcase the capabilities of these simulators. +Those environment collections are now deprecated in favor of Isaac Lab. + +Isaac Lab supersedes `IsaacGymEnvs`_, `OmniIsaacGymEnvs`_, and `Orbit`_ as the single robot +learning framework for Isaac Sim. It retains full access to the PhysX/Isaac Sim stack while +adding the Newton physics backend for kit-less deployments, an expanded sensor suite, imitation +learning tooling, XR teleoperation, and a rich set of pre-built tasks. Is Isaac Lab a simulator? ------------------------- -Often, when people think of simulators, they think of various commonly available engines, such as -`MuJoCo`_, `Bullet`_, and `Flex`_. These engines are powerful and have been used in a number of -research projects. However, they are not designed to be a general purpose simulator for robotics. -Rather they are primarily physics engines that are used to simulate the dynamics of rigid and -deformable bodies. They are shipped with some basic rendering capabilities to visualize the -simulation and provide parsing capabilities of different scene description formats. - -Various recent works combine these physics engines with different rendering engines to provide -a more complete simulation environment. They include APIs that allow reading and writing to the -physics and rendering engines. In some cases, they support ROS and hardware-in-the-loop simulation -for more robotic-specific applications. An example of these include `AirSim`_, `DoorGym`_, `ManiSkill`_, -`ThreeDWorld`_ and lastly, `Isaac Sim`_. - -At its core, Isaac Lab is **not** a robotics simulator, but a framework for building robot learning -applications on top of `Isaac Sim`_ and `Newton`_. An equivalent example of such a framework is `RoboSuite`_, which -is built on top of `MuJoCo`_ and is specific to fixed-base robots. Other examples include -`MuJoCo Playground`_ and `Isaac Gym`_ which use `MJX`_ and `PhysX`_ respectively. Isaac Lab supports -both `PhysX`_ and `Newton`_ as physics backends. They -include a number of pre-built tasks with separated out stand-alone implementations for individual -tasks. While this is a good starting point (and often convenient), a lot of code -repetition occurs across different task implementations, which can reduce code-reuse for larger -projects and teams. - -The main goal of Isaac Lab is to provide a unified framework for robot learning that includes -a variety of tooling and features that are required for robot learning, while being easy to -use and extend. It includes design patterns that simplify many of the common requirements for -robotics research. These include simulating sensors at different frequencies, connecting to different -teleoperation interfaces for data collection, switching action spaces for policy learning, -using Hydra for configuration management, supporting different learning libraries and more. -Isaac Lab supports designing tasks using *manager-based (modularized)* and *direct (single-script -similar to Isaac Gym)* patterns, leaving it up to the user to choose the best approach for their -use-case. For each of these patterns, Isaac Lab includes a number of pre-built tasks that can be -used for benchmarking and research. +At its core, Isaac Lab is **not** a robotics simulator; it is a framework for building robot +learning applications on top of a simulator. An analogous example is `RoboSuite`_, which is +built on top of `MuJoCo`_ for fixed-base manipulation. Other examples include +`MuJoCo Playground`_ (built on `MJX`_) and Isaac Gym (built on `PhysX`_). + +Isaac Lab supports both `PhysX`_ and `Newton`_ as physics backends and is deliberately +agnostic to the underlying engine — user environments and tasks do not import backend-specific +modules directly. + +The framework addresses a recurring problem with standalone task implementations: because each +task reimplements the observation, reward, termination, and randomization logic from scratch, +large projects accumulate significant code duplication. Isaac Lab solves this with two +complementary patterns: + +* **Manager-based** environments defer every behavioral concern to typed, composable + *manager* objects (``ObservationManager``, ``RewardManager``, ``TerminationManager``, + ``EventManager``, ``CurriculumManager``, ``CommandManager``, ``ActionManager``, + ``RecorderManager``). Each manager is driven by small, reusable MDP term functions that live + in :mod:`isaaclab.envs.mdp`. This makes it easy to mix and match terms across tasks and to + test individual components in isolation. + +* **Direct** environments implement ``_get_observations``, ``_get_rewards``, + ``_get_dones``, and ``_reset_idx`` directly in a subclass, similar to the Isaac Gym style. + They sacrifice some modularity for simplicity and are a natural starting point for rapid + prototyping. + +Both patterns expose a standard `gymnasium`_ ``Env`` interface with vectorized semantics, +so the same environment works unmodified with any of the supported RL libraries. +Configuration management uses `Hydra`_ with a preset system that allows selecting physics +backends and hyperparameter sweeps from the command line. Why should I use Isaac Lab? --------------------------- -Isaac Lab provides an open-sourced platform for the community to drive progress with consolidated efforts -toward designing benchmarks and robot learning systems as a joint initiative. This allows us to reuse -existing components and algorithms, and to build on top of each other's work. Doing so not only saves -time and effort, but also allows us to focus on the more important aspects of research. Our hope with -Isaac Lab is that it becomes the de-facto platform for robot learning research and an environment *zoo* -that leverages Isaac Sim. As the framework matures, we foresee it benefitting hugely from the latest -simulation developments (as part of internal developments at NVIDIA and collaborating partners) -and research in robotics. - -We are already working with labs in universities and research institutions to integrate their work into Isaac Lab -and hope that others in the community will join us too in this effort. If you are interested in contributing -to Isaac Lab, please reach out to us. +Isaac Lab provides an open-sourced platform for the community to build benchmarks and robot +learning systems together. Sharing a common infrastructure lets teams reuse existing components, +compare results on the same tasks, and focus on the research problems that matter rather than +rebuilding simulation scaffolding from scratch. + +Concretely, Isaac Lab offers: + +* **Two authoring patterns** — manager-based for modular research and direct for rapid + prototyping — with a shared :class:`~isaaclab.scene.InteractiveScene` and sensor stack. +* **Multi-backend simulation** — switch between PhysX and Newton from the command line; the + same environment code runs on both. +* **Rich sensor suite** — cameras (tiled and standard), ray-casters, contact sensors, IMU, + frame transformers, joint-wrench sensors, and visuo-tactile sensors. +* **Imitation learning tooling** — ``isaaclab_mimic`` provides cuRobo-based planners and a + full dataset-generation pipeline for human demonstration collection. +* **Teleoperation and XR** — ``isaaclab_teleop`` supports OpenXR, CloudXR, gamepads, + spacemouses, and Haply devices with retargeters for manipulators and humanoids. +* **Hydra configuration management** — hierarchical configs with command-line overrides and a + preset system for multi-backend environment variants. +* **RL library integrations** — wrappers for RSL-RL, skrl, Stable Baselines 3, and RL Games + ship in ``isaaclab_rl``. +* **Kit-less deployment** — run policies and simulations using the Newton backend without a + full Isaac Sim installation. + +We are working with labs in universities and research institutions to integrate their work into +Isaac Lab and hope that others in the community will join us. If you are interested in +contributing, please reach out to us. .. _PhysX: https://developer.nvidia.com/physx-sdk .. _Newton: https://github.com/newton-physics/newton +.. _Warp: https://github.com/NVIDIA/warp .. _Isaac Sim: https://developer.nvidia.com/isaac-sim +.. _Omniverse: https://www.nvidia.com/en-us/omniverse/ .. _Isaac Gym: https://developer.nvidia.com/isaac-gym .. _IsaacGymEnvs: https://github.com/isaac-sim/IsaacGymEnvs .. _OmniIsaacGymEnvs: https://github.com/isaac-sim/OmniIsaacGymEnvs .. _Orbit: https://isaac-orbit.github.io/ .. _Isaac Automator: https://github.com/isaac-sim/IsaacAutomator +.. _gymnasium: https://gymnasium.farama.org/ +.. _Hydra: https://hydra.cc/ +.. _RSL-RL: https://github.com/leggedrobotics/rsl_rl +.. _skrl: https://skrl.readthedocs.io/ +.. _Stable Baselines 3: https://stable-baselines3.readthedocs.io/ +.. _RL Games: https://github.com/Denys88/rl_games .. _AirSim: https://microsoft.github.io/AirSim/ .. _DoorGym: https://github.com/PSVL/DoorGym/ .. _ManiSkill: https://github.com/haosulab/ManiSkill