From 51a5519d0d175ae1e834bd2aca29d034cabe0fa5 Mon Sep 17 00:00:00 2001 From: Reima Karhila Date: Mon, 8 Dec 2025 14:51:47 +0200 Subject: [PATCH 1/2] Switching dsr1 fp8 to lmsys images for MI355, MI325 and MI300 --- .github/configs/amd-master.yaml | 6 +++--- benchmarks/dsr1_fp8_mi355x_docker.sh | 5 +++++ benchmarks/dsr1_fp8_mi355x_slurm.sh | 6 +++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index a2674153a..8d640fe31 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -22,7 +22,7 @@ dsr1-fp4-mi355x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } dsr1-fp8-mi300x-sglang: - image: rocm/7.0:rocm7.0_ubuntu_22.04_sgl-dev-v0.5.2-rocm7.0-mi30x-20250915 + image: lmsysorg/sglang:v0.5.5.post3-rocm700-mi30x model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: mi300x @@ -44,7 +44,7 @@ dsr1-fp8-mi300x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } dsr1-fp8-mi325x-sglang: - image: rocm/7.0:rocm7.0_ubuntu_22.04_sgl-dev-v0.5.2-rocm7.0-mi30x-20250915 + image: lmsysorg/sglang:v0.5.5.post3-rocm700-mi30x model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: mi325x @@ -66,7 +66,7 @@ dsr1-fp8-mi325x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } dsr1-fp8-mi355x-sglang: - image: rocm/7.0:rocm7.0_ubuntu_22.04_sgl-dev-v0.5.2-rocm7.0-mi35x-20250915 + image: lmsysorg/sglang:v0.5.5.post3-rocm700-mi35x model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: mi355x diff --git a/benchmarks/dsr1_fp8_mi355x_docker.sh b/benchmarks/dsr1_fp8_mi355x_docker.sh index 8c5038cee..573d1b6a4 100644 --- a/benchmarks/dsr1_fp8_mi355x_docker.sh +++ b/benchmarks/dsr1_fp8_mi355x_docker.sh @@ -14,10 +14,14 @@ # https://rocm.docs.amd.com/en/docs-7.0-docker/benchmark-docker/inference-sglang-deepseek-r1-fp8.html export SGLANG_USE_AITER=1 +export RCCL_MSCCL_ENABLE=0 +export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 + SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log) python3 -m sglang.launch_server \ + --attention-backend aiter \ --model-path $MODEL \ --host=0.0.0.0 \ --port $PORT \ @@ -27,6 +31,7 @@ python3 -m sglang.launch_server \ --mem-fraction-static 0.8 --disable-radix-cache \ --num-continuous-decode-steps 4 \ --max-prefill-tokens 196608 \ + --enable-torch-compile \ --cuda-graph-max-bs 128 > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/benchmarks/dsr1_fp8_mi355x_slurm.sh b/benchmarks/dsr1_fp8_mi355x_slurm.sh index 921f08a4c..8795d61b5 100644 --- a/benchmarks/dsr1_fp8_mi355x_slurm.sh +++ b/benchmarks/dsr1_fp8_mi355x_slurm.sh @@ -12,11 +12,14 @@ export HF_MODULES_CACHE="/tmp/hf_modules_cache/" export SGLANG_USE_AITER=1 +export RCCL_MSCCL_ENABLE=0 +export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log) set -x python3 -m sglang.launch_server \ + --attention-backend aiter \ --model-path $MODEL \ --host=0.0.0.0 \ --port $PORT \ @@ -27,7 +30,8 @@ python3 -m sglang.launch_server \ --disable-radix-cache \ --num-continuous-decode-steps 4 \ --max-prefill-tokens 196608 \ - --cuda-graph-max-bs 128 > $SERVER_LOG 2>&1 & + --cuda-graph-max-bs 128 \ + --enable-torch-compile > $SERVER_LOG 2>&1 & SERVER_PID=$! From 9fe1942df32f2917692bf05157e966107a3ba305 Mon Sep 17 00:00:00 2001 From: Reima Karhila Date: Mon, 8 Dec 2025 14:54:54 +0200 Subject: [PATCH 2/2] formatting --- benchmarks/dsr1_fp8_mi355x_docker.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/dsr1_fp8_mi355x_docker.sh b/benchmarks/dsr1_fp8_mi355x_docker.sh index 573d1b6a4..23eedf07e 100644 --- a/benchmarks/dsr1_fp8_mi355x_docker.sh +++ b/benchmarks/dsr1_fp8_mi355x_docker.sh @@ -17,7 +17,6 @@ export SGLANG_USE_AITER=1 export RCCL_MSCCL_ENABLE=0 export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log) python3 -m sglang.launch_server \