From aa93ff2f44f4c8f0c6f9fc57af282e1786d53692 Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 24 Mar 2026 22:45:25 -0500 Subject: [PATCH 1/3] feat: enhance Qwen benchmark scripts with additional parameters * Added CONTEXT_LENGTH and MAX_PREFILL_TOKENS variables for better configuration. * Updated launch_server command with new options: --tokenizer-worker-num, --enable-aiter-allreduce-fusion, --cuda-graph-max-bs, --context-length, --disable-radix-cache, --max-prefill-tokens, and --scheduler-recv-interval. --- benchmarks/single_node/qwen3.5_bf16_mi355x.sh | 9 +++++++++ benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/benchmarks/single_node/qwen3.5_bf16_mi355x.sh b/benchmarks/single_node/qwen3.5_bf16_mi355x.sh index f77390707..fb5550d0f 100755 --- a/benchmarks/single_node/qwen3.5_bf16_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_bf16_mi355x.sh @@ -19,6 +19,8 @@ hf download "$MODEL" SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} +CONTEXT_LENGTH=$((ISL + OSL + 20)) +MAX_PREFILL_TOKENS=32768 # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -30,6 +32,13 @@ python3 -m sglang.launch_server \ --port $PORT \ --tensor-parallel-size $TP \ --trust-remote-code \ + --tokenizer-worker-num 6 \ + --enable-aiter-allreduce-fusion \ + --cuda-graph-max-bs $CONC \ + --context-length $CONTEXT_LENGTH \ + --disable-radix-cache \ + --max-prefill-tokens $MAX_PREFILL_TOKENS \ + --scheduler-recv-interval 30 \ --mem-fraction-static 0.8 > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index f77390707..fb5550d0f 100644 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -19,6 +19,8 @@ hf download "$MODEL" SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} +CONTEXT_LENGTH=$((ISL + OSL + 20)) +MAX_PREFILL_TOKENS=32768 # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -30,6 +32,13 @@ python3 -m sglang.launch_server \ --port $PORT \ --tensor-parallel-size $TP \ --trust-remote-code \ + --tokenizer-worker-num 6 \ + --enable-aiter-allreduce-fusion \ + --cuda-graph-max-bs $CONC \ + --context-length $CONTEXT_LENGTH \ + --disable-radix-cache \ + --max-prefill-tokens $MAX_PREFILL_TOKENS \ + --scheduler-recv-interval 30 \ --mem-fraction-static 0.8 > $SERVER_LOG 2>&1 & SERVER_PID=$! From 8527313b2a22b551e36f4e304871c162e9c28905 Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 24 Mar 2026 22:48:55 -0500 Subject: [PATCH 2/3] Update perf-changelog.yaml to include new Qwen3.5 FP8 and BF16 SGLang benchmark configurations for MI355X, enhancing performance with updated CLI arguments. --- perf-changelog.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1a19fd6a5..b7db494db 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1055,3 +1055,10 @@ - "Enable VLLM_USE_FLASHINFER_MOE_INT4=1 for Kimi K2.5 INT4 B200 benchmark" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/935 +- config-keys: + - qwen3.5-fp8-mi355x-sglang + - qwen3.5-bf16-mi355x-sglang + description: + - "Update cli args of Qwen3.5 FP8 and BF16 SGLang benchmarks for MI355X to achieve better performance" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/942 + From 560edba74fce12c203eda71d07e82a8c45d28b55 Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Wed, 25 Mar 2026 04:49:14 -0500 Subject: [PATCH 3/3] Update Qwen benchmark scripts to replace --enable-aiter-allreduce-fusion with --enable-torch-compile for improved performance in BF16 and FP8 configurations. --- benchmarks/single_node/qwen3.5_bf16_mi355x.sh | 2 +- benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/qwen3.5_bf16_mi355x.sh b/benchmarks/single_node/qwen3.5_bf16_mi355x.sh index fb5550d0f..e339c9658 100755 --- a/benchmarks/single_node/qwen3.5_bf16_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_bf16_mi355x.sh @@ -33,7 +33,7 @@ python3 -m sglang.launch_server \ --tensor-parallel-size $TP \ --trust-remote-code \ --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ + --enable-torch-compile \ --cuda-graph-max-bs $CONC \ --context-length $CONTEXT_LENGTH \ --disable-radix-cache \ diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index fb5550d0f..e339c9658 100644 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -33,7 +33,7 @@ python3 -m sglang.launch_server \ --tensor-parallel-size $TP \ --trust-remote-code \ --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ + --enable-torch-compile \ --cuda-graph-max-bs $CONC \ --context-length $CONTEXT_LENGTH \ --disable-radix-cache \