From 2b24d8d5cf93b0135c2eb5545a68ffeeb3cf5a9e Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Mon, 25 Aug 2025 23:49:01 -0700 Subject: [PATCH] Fixes distributed training hanging issue --- apps/isaaclab.python.headless.rendering.kit | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/isaaclab.python.headless.rendering.kit b/apps/isaaclab.python.headless.rendering.kit index 09fc3ba98efe..ed20ad42c321 100644 --- a/apps/isaaclab.python.headless.rendering.kit +++ b/apps/isaaclab.python.headless.rendering.kit @@ -83,6 +83,9 @@ app.updateOrder.checkForHydraRenderComplete = 1000 app.renderer.waitIdle=true app.hydraEngine.waitIdle=true +# Forces serial processing for omni graph to avoid NCCL timeout hangs in distributed training +app.execution.debug.forceSerial = true + app.audio.enabled = false # Enable Vulkan - avoids torch+cu12 error on windows