SGL GB300 Day 0 DSV4 FP4 disagg

.github/configs/nvidia-master.yaml

@@ -7722,3 +7722,38 @@ dsv4-fp4-gb200-dynamo-vllm:
         tp: 8
         ep: 8
         dp-attn: true
+
+dsv4-fp4-gb300-dynamo-sglang:
+  # _arm64 variant: GH runner pod doing `enroot import` is amd64, but
+  # gb300-cw compute nodes are aarch64 (Grace). Without the explicit
+  # arm64 tag the registry serves the amd64 manifest, which fails to
+  # exec on the compute side.
+  image: lmsysorg/sglang:deepseek-v4-grace-blackwell_arm64
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: gb300-cw
+  precision: fp4
+  framework: dynamo-sglang
+  multinode: true
+  disagg: true
+  # Uses the sa-bench recipe copied exactly from NVIDIA/srt-slurm:
+  # recipes/dsv4-pro/sglang/gb200-fp4/1k1k/disagg/stp/disagg-1p3d-tp8.yaml
+  # at commit 9d75f82acec163594658a440f39dd7f1bd35bd16.
+  seq-len-configs:
+  - isl: 1024
+    osl: 1024
+    search-space:
+    # 1 prefill worker and 3 decode workers, each TP=8.
+    - conc-list: [32, 64, 128, 256, 512, 1024]
+      prefill:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "CONFIG_FILE=recipes/dsv4-pro/sglang/gb200-fp4/1k1k/disagg/stp/disagg-1p3d-tp8.yaml"
+      decode:
+        num-worker: 3
+        tp: 8
+        ep: 1
+        dp-attn: false

.github/configs/runners.yaml

@@ -139,3 +139,8 @@ gb300:
 - 'gb300-nv_0'
 - 'gb300-nv_1'
 - 'gb300-nv_2'
+gb300-cw:
+- 'gb300-cw_0'
+- 'gb300-cw_1'
+- 'gb300-cw_2'
+- 'gb300-cw_3'

.github/workflows/benchmark-multinode-tmpl.yml

@@ -171,6 +171,9 @@ jobs:
           set -x
           # Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled
           echo "RESULT_FILENAME=${RESULT_FILENAME}" >> $GITHUB_ENV
+          rm -f multinode_server_logs.tar.gz
+          rm -rf LOGS
+          rm -f ${RESULT_FILENAME}_*.json agg_${RESULT_FILENAME}_*.json
 
           export ${{ join(fromJson(inputs.prefill-additional-settings), ' ') }} ${{ join(fromJson(inputs.decode-additional-settings), ' ') }}
           export IS_MULTINODE=true

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/gb200-fp4/1k1k/disagg/stp/disagg-1p1d-tp8.yaml

@@ -0,0 +1,80 @@
+# DeepSeek-V4-Pro disaggregated on GB200 (1P3D, TP=8, MXFP4)
+#
+# AIME 2025 (aime25): all 30 problems, full concurrency
+
+name: "dsv4-pro-gb200-1k1k-disagg-1p1d-tp8-aime"
+
+dynamo:
+  hash: "9d3c913d300eb368cda28b3f98a23a5762621e0d"
+
+frontend:
+  type: dynamo
+  nginx_container: nginx
+
+model:
+  path: "dspro"
+  container: "dspro-0426-nixl"
+  precision: "mxfp4"
+
+resources:
+  gpu_type: "gb200"
+  prefill_nodes: 2
+  decode_nodes: 2
+  prefill_workers: 1
+  decode_workers: 1
+  gpus_per_node: 4
+
+health_check:
+  max_attempts: 360
+  interval_seconds: 10
+
+backend:
+  type: sglang
+
+  prefill_environment:
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
+    SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0"
+
+  decode_environment:
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
+    SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0"
+
+  sglang_config:
+    prefill:
+      disaggregation-bootstrap-port: 30001
+      served-model-name: "dspro"
+      trust-remote-code: true
+      tensor-parallel-size: 8
+      disaggregation-mode: "prefill"
+      disaggregation-transfer-backend: nixl
+      moe-runner-backend: "flashinfer_mxfp4"
+      chunked-prefill-size: 8192
+      disable-flashinfer-autotune: true
+      max-running-requests: 1024
+      cuda-graph-max-bs: 2048
+      mem-fraction-static: 0.85
+
+    decode:
+      served-model-name: "dspro"
+      disaggregation-bootstrap-port: 30001
+      trust-remote-code: true
+      tensor-parallel-size: 8
+      disaggregation-mode: "decode"
+      disaggregation-transfer-backend: nixl
+      moe-runner-backend: "flashinfer_mxfp4"
+      chunked-prefill-size: 8192
+      disable-flashinfer-autotune: true
+      max-running-requests: 1024
+      cuda-graph-max-bs: 1024
+      mem-fraction-static: 0.85
+
+benchmark:
+  type: "aime"
+  aime_dataset: "aime25"
+  num_threads: 30

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/gb200-fp4/1k1k/disagg/stp/disagg-1p3d-tp8.yaml

@@ -0,0 +1,83 @@
+# DeepSeek-V4-Pro disaggregated on GB200 (1P3D, TP=8, MXFP4)
+#
+# Some basic rate matching
+# TODO: no optimizations have been applied yet
+
+dynamo:
+  hash: "9d3c913d300eb368cda28b3f98a23a5762621e0d"
+
+frontend:
+  type: dynamo
+  nginx_container: nginx
+
+model:
+  path: "dspro"
+  container: "dspro-0426"
+  precision: "mxfp4"
+
+resources:
+  gpu_type: "gb200"
+  prefill_nodes: 2
+  decode_nodes: 6
+  prefill_workers: 1
+  decode_workers: 3
+  gpus_per_node: 4
+
+health_check:
+  max_attempts: 360
+  interval_seconds: 10
+
+backend:
+  type: sglang
+
+  prefill_environment:
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
+    SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0"
+
+  decode_environment:
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
+    SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0"
+
+  sglang_config:
+    prefill:
+      disaggregation-bootstrap-port: 30001
+      served-model-name: "dspro"
+      trust-remote-code: true
+      tensor-parallel-size: 8
+      disaggregation-mode: "prefill"
+      disaggregation-transfer-backend: nixl
+      moe-runner-backend: "flashinfer_mxfp4"
+      chunked-prefill-size: 8192
+      disable-flashinfer-autotune: true
+      max-running-requests: 1024
+      cuda-graph-max-bs: 2048
+      mem-fraction-static: 0.85
+
+    decode:
+      served-model-name: "dspro"
+      disaggregation-bootstrap-port: 30001
+      trust-remote-code: true
+      tensor-parallel-size: 8
+      disaggregation-mode: "decode"
+      disaggregation-transfer-backend: nixl
+      moe-runner-backend: "flashinfer_mxfp4"
+      chunked-prefill-size: 8192
+      disable-flashinfer-autotune: true
+      max-running-requests: 1024
+      cuda-graph-max-bs: 1024
+      mem-fraction-static: 0.85
+
+benchmark:
+  type: "sa-bench"
+  isl: 1024
+  osl: 1024
+  random_range_ratio: 0.8
+  concurrencies: "32x64x128x256x512x1024"
+  req_rate: "inf"
+  use_chat_template: false

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/gb200-fp4/8k1k/disagg/stp/disagg-1p1d-tp8.yaml

@@ -0,0 +1,94 @@
+# DeepSeek-V4-Pro disaggregated on GB200 (1P1D, TP=8, MXFP4) — 8k1k newtp + dspro-0426.
+# WIP
+
+name: "gb200-mxfp4-8k1k-disagg-newtp"
+
+dynamo:
+  hash: "9d3c913d300eb368cda28b3f98a23a5762621e0d"
+
+frontend:
+  type: dynamo
+  nginx_container: nginx
+
+model:
+  path: "dspro"
+  container: "dspro-0426"
+  precision: "mxfp4"
+
+resources:
+  gpu_type: "gb200"
+  prefill_nodes: 2
+  decode_nodes: 2
+  prefill_workers: 1
+  decode_workers: 1
+  gpus_per_node: 4
+
+health_check:
+  max_attempts: 360
+  interval_seconds: 10
+
+backend:
+  type: sglang
+
+  prefill_environment:
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
+    SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0"
+    SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
+    SGLANG_OPT_USE_JIT_NORM: "1"
+    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
+    SGLANG_OPT_USE_TOPK_V2: "1"
+
+  decode_environment:
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
+    SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0"
+    SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
+    SGLANG_OPT_USE_JIT_NORM: "1"
+    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
+    SGLANG_OPT_USE_TOPK_V2: "1"
+
+  sglang_config:
+    prefill:
+      disaggregation-bootstrap-port: 30001
+      served-model-name: "dspro"
+      trust-remote-code: true
+      tensor-parallel-size: 8
+      disaggregation-mode: "prefill"
+      disaggregation-transfer-backend: nixl
+      moe-runner-backend: "flashinfer_mxfp4"
+      chunked-prefill-size: 8192
+      disable-flashinfer-autotune: true
+      mem-fraction-static: 0.90
+      max-running-requests: 512
+      cuda-graph-max-bs: 512
+      swa-full-tokens-ratio: 0.1
+
+    decode:
+      served-model-name: "dspro"
+      disaggregation-bootstrap-port: 30001
+      trust-remote-code: true
+      tensor-parallel-size: 8
+      disaggregation-mode: "decode"
+      disaggregation-transfer-backend: nixl
+      moe-runner-backend: "flashinfer_mxfp4"
+      chunked-prefill-size: 8192
+      disable-flashinfer-autotune: true
+      mem-fraction-static: 0.90
+      max-running-requests: 512
+      cuda-graph-max-bs: 512
+      swa-full-tokens-ratio: 0.1
+
+benchmark:
+  type: "sa-bench"
+  isl: 8192
+  osl: 1024
+  random_range_ratio: 0.8
+  concurrencies: "32x64x128x256x512"
+  req_rate: "inf"
+  use_chat_template: true
+  custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"

perf-changelog.yaml

@@ -1877,7 +1877,7 @@
     - "Image pinned to lmsysorg/sglang:deepseek-v4-b300@sha256:26e116bd211e300dbb76924d56c5cbe6cc3ee5ee2fe314859cb8774f5bc070f3"
     - "DP-attention path enables SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 for better SWA eviction behavior"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1185
-  
+
 - config-keys:
     - dsv4-fp4-b200-sglang
   description:
@@ -1985,3 +1985,12 @@
     - "Topology: 1 prefill DEP8 worker and 4 decode TP8 workers with dedicated NATS/etcd"
     - "Mirrors the historical 1P4D DEP8/TP8 offload point from srt-slurm aflowers/vllm-gb200-v0.20.0"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1218
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-sglang
+  description:
+    - "Add DeepSeek-V4-Pro FP4 GB300 Dynamo SGLang disaggregated multinode configuration"
+    - "Image: lmsysorg/sglang:deepseek-v4-grace-blackwell_arm64 (gb300-cw compute nodes are aarch64)"
+    - "Topology: 1 prefill worker + 3 decode workers, TP=8, MXFP4 MoE kernels, NIXL KV transfer"
+    - "Recipes copied exactly from NVIDIA/srt-slurm recipes/dsv4-pro/sglang/gb200-fp4 at commit 9d75f82acec163594658a440f39dd7f1bd35bd16"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1169