From 08b656f0aa24481ac593d25927d04ce8bf4dee19 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 15 May 2026 17:06:03 +0000 Subject: [PATCH] [Fix] Pin numpy!=2.3.5 to dodge OpenBLAS atfork SIGSEGV at Kit fork() NumPy 2.3.5 ships a vendored OpenBLAS (libscipy_openblas64_-fdde5778.so) whose pthread_atfork handler calls blas_thread_shutdown_ -> pthread_join on workers that don't exist in the child of fork(). Kit's libomni.platforminfo calls fork() during SimulationApp startup, which triggers the handler and SIGSEGVs the test process. IsaacLab's CI Docker layer (docker/Dockerfile.base:117) runs 'isaaclab.sh --install' which pip-installs the source packages and resolves numpy>=2 against the IsaacLab dep tree. pin-pink -> pinocchio (pin) -> cmeel-boost transitively caps numpy <2.4, so pip picks the highest 2.3.x: 2.3.5. That landed the broken OpenBLAS into site-packages, shadowing Isaac Sim's prebundled numpy 2.3.1. Tightening to 'numpy>=2,!=2.3.5' across the four packages that declare a numpy dep (isaaclab, isaaclab_tasks, isaaclab_rl, isaaclab_visualizers) keeps the loose lower bound but excludes the single known-broken release. Pip resolves to 2.3.4, which ships a different bundled OpenBLAS hash (libscipy_openblas64_-8fb3d286.so) that was the resolved version for IsaacLab CI prior to numpy 2.3.5 without these crashes. Refs: - numpy/numpy#30092 - scipy/scipy#23686 - OpenMathLib/OpenBLAS#5520 - JIRA OMPE-92261 Verified locally: - numpy 2.3.0 / 2.3.1 ship libscipy_openblas64_-56d6093b.so (safe) - numpy 2.3.2 / 2.3.3 / 2.3.4 ship libscipy_openblas64_-8fb3d286.so - numpy 2.3.5 ships libscipy_openblas64_-fdde5778.so (broken) - Isaac Sim base image (both 5/11 and 5/15 candidates) prebundles numpy 2.3.1 with -56d6093b at omni.kit.pip_archive/pip_prebundle A longer-term fix is bumping cmeel-boost upstream so numpy 2.4.1+ becomes resolvable; coordination with Isaac Sim base image is also in flight separately. --- .../changelog.d/jichuanh-pin-numpy-not-2-3-5.rst | 10 ++++++++++ source/isaaclab/setup.py | 6 +++++- .../changelog.d/jichuanh-pin-numpy-not-2-3-5.rst | 6 ++++++ source/isaaclab_rl/setup.py | 3 ++- .../changelog.d/jichuanh-pin-numpy-not-2-3-5.rst | 6 ++++++ source/isaaclab_tasks/setup.py | 3 ++- .../changelog.d/jichuanh-pin-numpy-not-2-3-5.rst | 6 ++++++ source/isaaclab_visualizers/setup.py | 3 ++- 8 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 source/isaaclab/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst create mode 100644 source/isaaclab_rl/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst create mode 100644 source/isaaclab_tasks/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst create mode 100644 source/isaaclab_visualizers/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst diff --git a/source/isaaclab/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst b/source/isaaclab/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst new file mode 100644 index 000000000000..14d6f415aee2 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst @@ -0,0 +1,10 @@ +Fixed +^^^^^ + +* Excluded NumPy 2.3.5 from the install constraint (``numpy>=2,!=2.3.5``). The 2.3.5 + release ships a vendored OpenBLAS (``libscipy_openblas64_-fdde5778.so``) whose + ``pthread_atfork`` handler crashes inside Kit's ``libomni.platforminfo`` ``fork()`` + during ``SimulationApp`` startup, manifesting as non-deterministic SIGSEGV in CI + test jobs. See `numpy#30092 `_ and + ``OMPE-92261``. With this change pip resolves to NumPy 2.3.4, which ships a + different OpenBLAS bundle without the regression. diff --git a/source/isaaclab/setup.py b/source/isaaclab/setup.py index 1f3be503574e..a6f3e2dc87fb 100644 --- a/source/isaaclab/setup.py +++ b/source/isaaclab/setup.py @@ -18,7 +18,11 @@ # Minimum dependencies required prior to installation INSTALL_REQUIRES = [ # generic - "numpy>=2", + # !=2.3.5 dodges the broken OpenBLAS bundle (libscipy_openblas64_-fdde5778.so) + # whose pthread_atfork handler SIGSEGVs inside Kit's libomni.platforminfo fork() + # during CI startup. See numpy/numpy#30092 and OMPE-92261. The pin-pink -> + # pinocchio -> cmeel-boost transitive constraint already caps numpy <2.4. + "numpy>=2,!=2.3.5", "torch>=2.10", "onnx>=1.18.0", # 1.16.2 throws access violation on Windows "prettytable==3.3.0", diff --git a/source/isaaclab_rl/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst b/source/isaaclab_rl/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst new file mode 100644 index 000000000000..ece20feff3db --- /dev/null +++ b/source/isaaclab_rl/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst @@ -0,0 +1,6 @@ +Fixed +^^^^^ + +* Tightened the NumPy install constraint to ``numpy>=2,!=2.3.5`` (was unconstrained) + to keep the package consistent with :file:`isaaclab/setup.py` and avoid the + OpenBLAS at-fork SIGSEGV. diff --git a/source/isaaclab_rl/setup.py b/source/isaaclab_rl/setup.py index 53255aa55b4f..5595bf22a659 100644 --- a/source/isaaclab_rl/setup.py +++ b/source/isaaclab_rl/setup.py @@ -19,7 +19,8 @@ # Minimum dependencies required prior to installation INSTALL_REQUIRES = [ # generic - "numpy", + # !=2.3.5 dodges the broken OpenBLAS bundle; see source/isaaclab/setup.py. + "numpy>=2,!=2.3.5", "torch>=2.10", "torchvision>=0.25.0", # ensure compatibility with torch 2.10.0 "protobuf>=4.25.8,!=5.26.0", diff --git a/source/isaaclab_tasks/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst b/source/isaaclab_tasks/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst new file mode 100644 index 000000000000..c4855e22d803 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst @@ -0,0 +1,6 @@ +Fixed +^^^^^ + +* Excluded NumPy 2.3.5 from the install constraint (``numpy>=2,!=2.3.5``) to match + :file:`isaaclab/setup.py` and avoid the OpenBLAS at-fork SIGSEGV during Kit + startup. diff --git a/source/isaaclab_tasks/setup.py b/source/isaaclab_tasks/setup.py index a719231ca986..18ca266aca73 100644 --- a/source/isaaclab_tasks/setup.py +++ b/source/isaaclab_tasks/setup.py @@ -18,7 +18,8 @@ # Minimum dependencies required prior to installation INSTALL_REQUIRES = [ # generic - "numpy>=2", + # !=2.3.5 dodges the broken OpenBLAS bundle; see source/isaaclab/setup.py. + "numpy>=2,!=2.3.5", "torch>=2.10", "torchvision>=0.25.0", # ensure compatibility with torch 2.10.0 "protobuf>=4.25.8,!=5.26.0", diff --git a/source/isaaclab_visualizers/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst b/source/isaaclab_visualizers/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst new file mode 100644 index 000000000000..ece20feff3db --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/jichuanh-pin-numpy-not-2-3-5.rst @@ -0,0 +1,6 @@ +Fixed +^^^^^ + +* Tightened the NumPy install constraint to ``numpy>=2,!=2.3.5`` (was unconstrained) + to keep the package consistent with :file:`isaaclab/setup.py` and avoid the + OpenBLAS at-fork SIGSEGV. diff --git a/source/isaaclab_visualizers/setup.py b/source/isaaclab_visualizers/setup.py index 008fe15c8d6c..aaa146dcf5cd 100644 --- a/source/isaaclab_visualizers/setup.py +++ b/source/isaaclab_visualizers/setup.py @@ -10,7 +10,8 @@ # Base requirements shared across visualizer backends. INSTALL_REQUIRES = [ "isaaclab", - "numpy", + # !=2.3.5 dodges the broken OpenBLAS bundle; see source/isaaclab/setup.py. + "numpy>=2,!=2.3.5", ] # Every Newton declaration in the repo must use the SAME extra spec (`newton[sim]`).