From 20ca1bc0323d11a409f2406b58da02ac6401b48d Mon Sep 17 00:00:00 2001 From: "Reima Karhila (AMD)" Date: Wed, 10 Dec 2025 12:33:44 +0200 Subject: [PATCH 1/7] Use upstream SGLang images on mi300, mi325 and mi355 for dsr1fp8 (#307) * Switching dsr1 fp8 to lmsys images for MI355, MI325 and MI300 --------- Co-authored-by: Chun Fang --- .github/configs/amd-master.yaml | 6 +++--- benchmarks/dsr1_fp8_mi355x_docker.sh | 4 ++++ benchmarks/dsr1_fp8_mi355x_slurm.sh | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index a2674153a..8d640fe31 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -22,7 +22,7 @@ dsr1-fp4-mi355x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } dsr1-fp8-mi300x-sglang: - image: rocm/7.0:rocm7.0_ubuntu_22.04_sgl-dev-v0.5.2-rocm7.0-mi30x-20250915 + image: lmsysorg/sglang:v0.5.5.post3-rocm700-mi30x model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: mi300x @@ -44,7 +44,7 @@ dsr1-fp8-mi300x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } dsr1-fp8-mi325x-sglang: - image: rocm/7.0:rocm7.0_ubuntu_22.04_sgl-dev-v0.5.2-rocm7.0-mi30x-20250915 + image: lmsysorg/sglang:v0.5.5.post3-rocm700-mi30x model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: mi325x @@ -66,7 +66,7 @@ dsr1-fp8-mi325x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } dsr1-fp8-mi355x-sglang: - image: rocm/7.0:rocm7.0_ubuntu_22.04_sgl-dev-v0.5.2-rocm7.0-mi35x-20250915 + image: lmsysorg/sglang:v0.5.5.post3-rocm700-mi35x model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: mi355x diff --git a/benchmarks/dsr1_fp8_mi355x_docker.sh b/benchmarks/dsr1_fp8_mi355x_docker.sh index 8c5038cee..23eedf07e 100644 --- a/benchmarks/dsr1_fp8_mi355x_docker.sh +++ b/benchmarks/dsr1_fp8_mi355x_docker.sh @@ -14,10 +14,13 @@ # https://rocm.docs.amd.com/en/docs-7.0-docker/benchmark-docker/inference-sglang-deepseek-r1-fp8.html export SGLANG_USE_AITER=1 +export RCCL_MSCCL_ENABLE=0 +export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log) python3 -m sglang.launch_server \ + --attention-backend aiter \ --model-path $MODEL \ --host=0.0.0.0 \ --port $PORT \ @@ -27,6 +30,7 @@ python3 -m sglang.launch_server \ --mem-fraction-static 0.8 --disable-radix-cache \ --num-continuous-decode-steps 4 \ --max-prefill-tokens 196608 \ + --enable-torch-compile \ --cuda-graph-max-bs 128 > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/benchmarks/dsr1_fp8_mi355x_slurm.sh b/benchmarks/dsr1_fp8_mi355x_slurm.sh index 921f08a4c..8795d61b5 100644 --- a/benchmarks/dsr1_fp8_mi355x_slurm.sh +++ b/benchmarks/dsr1_fp8_mi355x_slurm.sh @@ -12,11 +12,14 @@ export HF_MODULES_CACHE="/tmp/hf_modules_cache/" export SGLANG_USE_AITER=1 +export RCCL_MSCCL_ENABLE=0 +export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log) set -x python3 -m sglang.launch_server \ + --attention-backend aiter \ --model-path $MODEL \ --host=0.0.0.0 \ --port $PORT \ @@ -27,7 +30,8 @@ python3 -m sglang.launch_server \ --disable-radix-cache \ --num-continuous-decode-steps 4 \ --max-prefill-tokens 196608 \ - --cuda-graph-max-bs 128 > $SERVER_LOG 2>&1 & + --cuda-graph-max-bs 128 \ + --enable-torch-compile > $SERVER_LOG 2>&1 & SERVER_PID=$! From e5a824fb00a230ec991951cecfde34face3abf72 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 15 Dec 2025 16:20:33 -0600 Subject: [PATCH 2/7] add changes to perf changelog --- perf-changelog.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 112145f10..b921c45bc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -81,3 +81,10 @@ - Update vLLM image for NVIDIA configs from vLLM 0.11.0 to vLLM 0.11.2 - Adds kv-cache-dtype: fp8 to benchmarks/gptoss_fp4_b200_docker.sh PR: https://github.com/InferenceMAX/InferenceMAX/pull/273 +- config-keys: + - dsr1-fp8-mi300x-sglang + - dsr1-fp8-mi325x-sglang + - dsr1-fp8-mi355x-sglang + description: | + - Use upstream SGLang images on mi300, mi325 and mi355 for dsr1fp8 + PR: https://github.com/InferenceMAX/InferenceMAX/pull/332 From aae7cdedbe2ad8c1eb642301694b21a5ff70720f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 30 Dec 2025 15:06:09 +0000 Subject: [PATCH 3/7] update perf changelog --- perf-changelog.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1be1afd04..2710b034f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -98,13 +98,6 @@ - "Add more configurations for GB200 SGLang DSR1" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/335 -- config-keys: - - dsr1-fp8-mi300x-sglang - - dsr1-fp8-mi325x-sglang - - dsr1-fp8-mi355x-sglang - description: | - - Use upstream SGLang images on mi300, mi325 and mi355 for dsr1fp8 - PR: https://github.com/InferenceMAX/InferenceMAX/pull/332 - config-keys: - dsr1-fp4-mi355x-sglang description: @@ -123,3 +116,11 @@ description: - "Updating MI355x Deepseek-R1 FP4 SGLang Image to upstream v0.5.6.post2" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/369 + +- config-keys: + - dsr1-fp8-mi300x-sglang + - dsr1-fp8-mi325x-sglang + - dsr1-fp8-mi355x-sglang + description: | + - Use upstream SGLang images on mi300, mi325 and mi355 for dsr1fp8 + PR: https://github.com/InferenceMAX/InferenceMAX/pull/307 From 2d0e88f171de77a715f2819ecf619b1ec31a2ebf Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 30 Dec 2025 15:07:38 +0000 Subject: [PATCH 4/7] update perf changelog pt 2 --- perf-changelog.yaml | 4 ++-- utils/collect_results.py | 4 +++- utils/summarize.py | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2710b034f..fe1a1ecc5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -121,6 +121,6 @@ - dsr1-fp8-mi300x-sglang - dsr1-fp8-mi325x-sglang - dsr1-fp8-mi355x-sglang - description: | + description: - Use upstream SGLang images on mi300, mi325 and mi355 for dsr1fp8 - PR: https://github.com/InferenceMAX/InferenceMAX/pull/307 + pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/307 diff --git a/utils/collect_results.py b/utils/collect_results.py index e97b7cca6..6bbaab441 100644 --- a/utils/collect_results.py +++ b/utils/collect_results.py @@ -10,7 +10,9 @@ for result_path in results_dir.rglob(f'*.json'): with open(result_path) as f: result = json.load(f) - agg_results.append(result) + # Skip aggregated result files (lists) and only process individual results (dicts) + if isinstance(result, dict) and 'is_multinode' in result: + agg_results.append(result) with open(f'agg_{exp_name}.json', 'w') as f: json.dump(agg_results, f, indent=2) diff --git a/utils/summarize.py b/utils/summarize.py index a46c2e02a..4c1c86db6 100644 --- a/utils/summarize.py +++ b/utils/summarize.py @@ -38,7 +38,9 @@ for result_path in results_dir.rglob('*.json'): with open(result_path) as f: result = json.load(f) - results.append(result) + # Skip aggregated result files (lists) and only process individual results (dicts) + if isinstance(result, dict) and 'is_multinode' in result: + results.append(result) single_node_results = [r for r in results if not r['is_multinode']] multinode_results = [r for r in results if r['is_multinode']] From 70f4b9044f3cdba16081a932ff81c64c1ac47f77 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 30 Dec 2025 15:08:29 +0000 Subject: [PATCH 5/7] update perf changelog pt 3 --- utils/collect_results.py | 4 +--- utils/summarize.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/utils/collect_results.py b/utils/collect_results.py index 6bbaab441..e97b7cca6 100644 --- a/utils/collect_results.py +++ b/utils/collect_results.py @@ -10,9 +10,7 @@ for result_path in results_dir.rglob(f'*.json'): with open(result_path) as f: result = json.load(f) - # Skip aggregated result files (lists) and only process individual results (dicts) - if isinstance(result, dict) and 'is_multinode' in result: - agg_results.append(result) + agg_results.append(result) with open(f'agg_{exp_name}.json', 'w') as f: json.dump(agg_results, f, indent=2) diff --git a/utils/summarize.py b/utils/summarize.py index 4c1c86db6..a46c2e02a 100644 --- a/utils/summarize.py +++ b/utils/summarize.py @@ -38,9 +38,7 @@ for result_path in results_dir.rglob('*.json'): with open(result_path) as f: result = json.load(f) - # Skip aggregated result files (lists) and only process individual results (dicts) - if isinstance(result, dict) and 'is_multinode' in result: - results.append(result) + results.append(result) single_node_results = [r for r in results if not r['is_multinode']] multinode_results = [r for r in results if r['is_multinode']] From 3b1299c83a6a241b72d5d40dc5e51e80ca1a65e7 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 5 Jan 2026 10:38:40 -0500 Subject: [PATCH 6/7] remove deleted links in .yaml --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7510f3def..01136d5ea 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -116,7 +116,7 @@ description: - "Updating MI355x Deepseek-R1 FP4 SGLang Image to upstream v0.5.6.post2" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/369 - + - config-keys: - dsr1-fp4-b200-sglang - dsr1-fp8-b200-sglang From 1324e9e0d2667a2252508e46795e866c52952879 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 5 Jan 2026 10:39:49 -0500 Subject: [PATCH 7/7] fix AMD's perf-changelog.yaml Updated the pull request link for dsr1fp8 in the changelog. --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 01136d5ea..c7f68885c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -140,4 +140,4 @@ - dsr1-fp8-mi355x-sglang description: - Use upstream SGLang images on mi300, mi325 and mi355 for dsr1fp8 - pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/307 + pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/332