From b422e5a9a063baa7c11360e62f02186ce2574d45 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 8 May 2026 16:46:37 +0000 Subject: [PATCH 1/2] Fix Newton viewer fully-black: assign PyVec3 to camera.pos NewtonVisualizer._apply_camera_pose was assigning ``self._viewer.camera.pos = wp.vec3(*cam_pos)``, but Newton's ``Camera.translate()`` adds a ``pyglet.math.Vec3`` delta to ``camera.pos`` via ``+=`` on every viewer update. warp 1.13's strict ``__add__`` rejects ``wp.vec3 + pyglet.math.Vec3`` with:: TypeError: Built-in functions cannot be called with non-Warp array types, such as lists, tuples, and NumPy arrays. Use a Warp type such as `wp.vec`, `wp.mat`, `wp.quat`, or `wp.transform`. The TypeError is silenced by the visualizer's ``try/except`` (``logger.debug``), which then short-circuits before ``renderer.render()`` -- so the framebuffer never gets written and ``ViewerGL.get_frame`` reads back all zeros. The test asserts that the last frame is non-black, hence "Viewer frame appears fully black." Same render-path failure manifested across the Newton 1.2.0rc2 + warp 1.13 cohort. Earlier hypotheses ruled out: viewer-code rc1->rc2 diff, ``wp.RegisteredGLBuffer`` API change, pure flakiness, the bump cohort alone, ``_make_current()``, and an explicit ``glFinish`` + ``wp.synchronize_device``. Direct CPU readback of the FBO confirmed it was empty (and a control ``glClear`` to red persisted untouched across all 60 frames -- proof that nothing was writing to the FBO, not even the renderer's own ``glClearColor`` + ``glClear``). Local repro on a non-MIG L40 with Kit 110.0.0 + Newton 1.2.0rc2 + warp 1.13.0 reproduces the failure deterministically. With ``cam_pos`` assigned as ``pyglet.math.Vec3`` instead, the ``+=`` is type-homogeneous, no exception, ``renderer.render()`` runs, and both ``[physx]`` and ``[newton]`` parametrizations pass in ~50 s each. This also re-enables the test that was skipped as a workaround in PR #5538. --- .../jichuanh-fix-newton-cam-pos-type.rst | 15 +++++++++++++++ .../newton/newton_visualizer.py | 12 +++++++++++- .../test/test_visualizer_cartpole_integration.py | 16 ---------------- 3 files changed, 26 insertions(+), 17 deletions(-) create mode 100644 source/isaaclab_visualizers/changelog.d/jichuanh-fix-newton-cam-pos-type.rst diff --git a/source/isaaclab_visualizers/changelog.d/jichuanh-fix-newton-cam-pos-type.rst b/source/isaaclab_visualizers/changelog.d/jichuanh-fix-newton-cam-pos-type.rst new file mode 100644 index 000000000000..6c6525acadf5 --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/jichuanh-fix-newton-cam-pos-type.rst @@ -0,0 +1,15 @@ +Fixed +^^^^^ + +* Fixed ``test_visualizer_cartpole_integration::test_cartpole_newton_visualizer_viewergl_rgb_motion`` + returning a fully-black ``ViewerGL.get_frame`` buffer on the Newton 1.2.0rc2 + + warp 1.13 cohort. ``NewtonVisualizer._apply_camera_pose`` was assigning + ``self._viewer.camera.pos = wp.vec3(*cam_pos)``, but Newton's + ``Camera.translate()`` adds a ``pyglet.math.Vec3`` delta with ``+=``. + warp 1.13's strict ``__add__`` rejects ``wp.vec3 + pyglet.math.Vec3`` + with ``TypeError``; the exception was silenced by the visualizer's + ``try/except``, which prevented ``renderer.render()`` from ever running + -- so the framebuffer stayed empty and read back as all zeros. The fix + assigns ``pyglet.math.Vec3`` instead, matching what Newton uses internally. +* Re-enabled ``test_cartpole_newton_visualizer_viewergl_rgb_motion`` after the + workaround skip in https://github.com/isaac-sim/IsaacLab/pull/5538. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py index b548a3e5f4f3..4d8a1335e6bb 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py @@ -463,7 +463,17 @@ def _apply_camera_pose(self, pose: tuple[tuple[float, float, float], tuple[float if self._viewer is None: return cam_pos, cam_target = pose - self._viewer.camera.pos = wp.vec3(*cam_pos) + # Newton's ``Camera.translate()`` adds a ``pyglet.math.Vec3`` delta to + # ``camera.pos`` via ``+=`` every viewer update. warp 1.13's strict + # ``__add__`` rejects ``wp.vec3 + pyglet.math.Vec3`` with + # ``TypeError: Built-in functions cannot be called with non-Warp array + # types``. The exception is silenced by the visualizer's try/except, + # which then skips ``renderer.render()`` -- the FBO is never written + # and the frame reads back fully black. Assigning a ``pyglet.math.Vec3`` + # here keeps the add homogeneous. + from pyglet.math import Vec3 as _PyVec3 + + self._viewer.camera.pos = _PyVec3(float(cam_pos[0]), float(cam_pos[1]), float(cam_pos[2])) cam_pos_np = np.array(cam_pos, dtype=np.float32) cam_target_np = np.array(cam_target, dtype=np.float32) direction = cam_target_np - cam_pos_np diff --git a/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py b/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py index 60f9921415cc..42d1368dcebf 100644 --- a/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py +++ b/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py @@ -522,22 +522,6 @@ def test_cartpole_newton_visualizer_tiled_camera_rgb_non_black( @pytest.mark.isaacsim_ci -@pytest.mark.skip( - reason=( - "ViewerGL.get_frame returns a fully-black 600x600x3 buffer in CI on the current " - "Isaac Sim image + Newton 1.2.0rc2 + warp-lang 1.13 cohort. Failure is " - "deterministic across two consecutive reruns of the same SHA and reproduces on " - "every PR that touches the rendering / camera / sensor / USD stack (5 PRs hit it " - "in the last 100 build.yaml runs); zero failures on PRs outside that scope. " - "Investigation ruled out: rc1->rc2 viewer code diff (7-line image_logger.clear " - "only), wp.RegisteredGLBuffer API (byte-identical 1.12 vs 1.13), pure flakiness " - "(deterministic), and the bump cohort alone (warp-1.12 branches both pass and " - "fail). Strongest remaining hypothesis: a CUDA-OpenGL interop init-order " - "fragility in the PBO + glReadPixels + RegisteredGLBuffer.map path that gets " - "tipped by any source change perturbing GL/CUDA bring-up. Re-enable once root " - "cause is identified." - ) -) @pytest.mark.parametrize("backend_kind", ["physx", "newton"]) def test_cartpole_newton_visualizer_viewergl_rgb_motion(backend_kind: str, caplog: pytest.LogCaptureFixture) -> None: """Newton GL (``ViewerGL.get_frame``): full motion steps, last frame non-black; early vs late differ; logs.""" From ea4db96dae1a1c3ea44baab94b06455794400cd4 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 8 May 2026 16:59:08 +0000 Subject: [PATCH 2/2] Hoist pyglet.math.Vec3 import to module level (review) --- .../newton/newton_visualizer.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py index 4d8a1335e6bb..aeafc29bd264 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py @@ -13,6 +13,7 @@ import numpy as np import warp as wp from newton.viewer import ViewerGL +from pyglet.math import Vec3 as PygletVec3 from isaaclab.visualizers.base_visualizer import BaseVisualizer @@ -463,17 +464,8 @@ def _apply_camera_pose(self, pose: tuple[tuple[float, float, float], tuple[float if self._viewer is None: return cam_pos, cam_target = pose - # Newton's ``Camera.translate()`` adds a ``pyglet.math.Vec3`` delta to - # ``camera.pos`` via ``+=`` every viewer update. warp 1.13's strict - # ``__add__`` rejects ``wp.vec3 + pyglet.math.Vec3`` with - # ``TypeError: Built-in functions cannot be called with non-Warp array - # types``. The exception is silenced by the visualizer's try/except, - # which then skips ``renderer.render()`` -- the FBO is never written - # and the frame reads back fully black. Assigning a ``pyglet.math.Vec3`` - # here keeps the add homogeneous. - from pyglet.math import Vec3 as _PyVec3 - - self._viewer.camera.pos = _PyVec3(float(cam_pos[0]), float(cam_pos[1]), float(cam_pos[2])) + # Match Newton's Camera native pos type: PyVec3, not wp.vec3. + self._viewer.camera.pos = PygletVec3(*cam_pos) cam_pos_np = np.array(cam_pos, dtype=np.float32) cam_target_np = np.array(cam_target, dtype=np.float32) direction = cam_target_np - cam_pos_np