From 8e0dfbf3dd485710dd6a149b57a2161125b34aac Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 16 Apr 2026 08:27:26 +0000 Subject: [PATCH 01/12] atom glm5 fp4 on mi355x Signed-off-by: seungrokj --- .../single_node/glm5_fp4_mi355x_atom.sh | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 benchmarks/single_node/glm5_fp4_mi355x_atom.sh diff --git a/benchmarks/single_node/glm5_fp4_mi355x_atom.sh b/benchmarks/single_node/glm5_fp4_mi355x_atom.sh new file mode 100644 index 000000000..31bc8b25f --- /dev/null +++ b/benchmarks/single_node/glm5_fp4_mi355x_atom.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + CONC \ + ISL \ + OSL \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME \ + EP_SIZE \ + DP_ATTENTION + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" + +SERVER_LOG=/workspace/server.log +PORT=${PORT:-8888} + +export OMP_NUM_THREADS=1 + +# Calculate max-model-len based on ISL and OSL +if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then + CALCULATED_MAX_MODEL_LEN="" +else + CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " +fi + +if [ "$EP_SIZE" -gt 1 ]; then + EP=" --enable-expert-parallel" +else + EP=" " +fi + +# Start GPU monitoring (power, temperature, clocks every second) +start_gpu_monitor + +set -x +pip install -U transformers +python3 -m atom.entrypoints.openai_server \ + --model $MODEL \ + --server-port $PORT \ + -tp $TP \ + --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ + --default-chat-template-kwargs '{"enable_thinking": false}' \ + --trust-remote-code \ + > $SERVER_LOG 2>&1 & + +SERVER_PID=$! + +# Wait for server to be ready +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +export PYTHONDONTWRITEBYTECODE=1 +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --trust-remote-code + +# After throughput, run evaluation only if RUN_EVAL is true +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +# Stop GPU monitoring +stop_gpu_monitor +set +x From 5c565eec429b56fa362efe88e9abb2e2e8c57647 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 16 Apr 2026 08:30:55 +0000 Subject: [PATCH 02/12] atom glm5 fp4 on mi355x Signed-off-by: seungrokj --- .github/configs/amd-master.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 89318004b..990c34692 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -277,6 +277,28 @@ glm5-fp8-mi355x-atom: search-space: - { tp: 8, conc-start: 4, conc-end: 256 } +glm5-fp4-mi355x-atom: + #TODO + image: TBD + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: mi355x + precision: fp4 + framework: atom + multinode: false + seq-len-configs: + #TODO: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 4, conc-start: 4, conc-end: 256 } + - { tp: 8, conc-start: 4, conc-end: 256 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-start: 4, conc-end: 256 } + - { tp: 8, conc-start: 4, conc-end: 256 } + kimik2.5-int4-mi355x-vllm: image: vllm/vllm-openai-rocm:v0.18.0 model: moonshotai/Kimi-K2.5 From a3b84c8878eee931030d9c69c1357343918fe38f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 16 Apr 2026 08:35:11 +0000 Subject: [PATCH 03/12] atom glm5 fp4 on mi355x Signed-off-by: seungrokj --- .github/configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 990c34692..efa2d7416 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -280,7 +280,7 @@ glm5-fp8-mi355x-atom: glm5-fp4-mi355x-atom: #TODO image: TBD - model: zai-org/GLM-5-FP8 + model: amd/GLM-5.1-MXFP4 model-prefix: glm5 runner: mi355x precision: fp4 From 2817c616855f014180ea9f76fb191828c83f75d6 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 23 Apr 2026 17:44:42 +0900 Subject: [PATCH 04/12] update glm5.1 fp4 mi355x atom config and benchmark script Co-Authored-By: Claude Sonnet 4.6 --- .github/configs/amd-master.yaml | 10 ++++------ ...m5_fp4_mi355x_atom.sh => glm5.1_fp4_mi355x_atom.sh} | 3 +++ 2 files changed, 7 insertions(+), 6 deletions(-) rename benchmarks/single_node/{glm5_fp4_mi355x_atom.sh => glm5.1_fp4_mi355x_atom.sh} (95%) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index efa2d7416..d1529109e 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -277,9 +277,8 @@ glm5-fp8-mi355x-atom: search-space: - { tp: 8, conc-start: 4, conc-end: 256 } -glm5-fp4-mi355x-atom: - #TODO - image: TBD +glm5.1-fp4-mi355x-atom: + image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post model: amd/GLM-5.1-MXFP4 model-prefix: glm5 runner: mi355x @@ -287,17 +286,16 @@ glm5-fp4-mi355x-atom: framework: atom multinode: false seq-len-configs: - #TODO: - isl: 1024 osl: 1024 search-space: + - { tp: 2, conc-start: 4, conc-end: 256 } - { tp: 4, conc-start: 4, conc-end: 256 } - - { tp: 8, conc-start: 4, conc-end: 256 } - isl: 8192 osl: 1024 search-space: + - { tp: 2, conc-start: 4, conc-end: 256 } - { tp: 4, conc-start: 4, conc-end: 256 } - - { tp: 8, conc-start: 4, conc-end: 256 } kimik2.5-int4-mi355x-vllm: image: vllm/vllm-openai-rocm:v0.18.0 diff --git a/benchmarks/single_node/glm5_fp4_mi355x_atom.sh b/benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh similarity index 95% rename from benchmarks/single_node/glm5_fp4_mi355x_atom.sh rename to benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh index 31bc8b25f..f1ff0207c 100644 --- a/benchmarks/single_node/glm5_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh @@ -39,6 +39,7 @@ fi # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor +MEM_FRAC_STATIC=0.9 set -x pip install -U transformers @@ -47,6 +48,8 @@ python3 -m atom.entrypoints.openai_server \ --server-port $PORT \ -tp $TP \ --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ + --max-num-seqs $CONC \ + --gpu-memory-utilization $MEM_FRAC_STATIC\ --default-chat-template-kwargs '{"enable_thinking": false}' \ --trust-remote-code \ > $SERVER_LOG 2>&1 & From b3eee3cd6ac8fa0c1bf0ab1ff29f5d3811a811a0 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 23 Apr 2026 17:48:11 +0900 Subject: [PATCH 05/12] add glm5.1 fp4 mi355x sglang config Co-Authored-By: Claude Sonnet 4.6 --- .github/configs/amd-master.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index d1529109e..b7a461281 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -277,6 +277,26 @@ glm5-fp8-mi355x-atom: search-space: - { tp: 8, conc-start: 4, conc-end: 256 } +glm5.1-fp4-mi355x-sglang: + image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260415 + model: amd/GLM-5.1-MXFP4 + model-prefix: glm5.1 + runner: mi355x + precision: fp4 + framework: sglang + multinode: false + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 2, conc-start: 4, conc-end: 256 } + - { tp: 4, conc-start: 4, conc-end: 16 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 2, conc-start: 4, conc-end: 256 } + - { tp: 4, conc-start: 4, conc-end: 16 } + glm5.1-fp4-mi355x-atom: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post model: amd/GLM-5.1-MXFP4 From 3938b4c15ec1d9fd837dae7c95212d8df1f97da0 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 23 Apr 2026 17:56:12 +0900 Subject: [PATCH 06/12] update perf-changelog with glm5.1 fp4 mi355x atom entry Co-Authored-By: Claude Sonnet 4.6 --- perf-changelog.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e27a2511a..ba69264d4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1717,3 +1717,20 @@ - "Trigger H100 multinode evals after NVSHEMM fixes" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1120 evals-only: true + +- config-keys: + - glm5.1-fp4-mi355x-atom + description: + - "Add GLM-5.1 MXFP4 single-node MI355X ATOM benchmark" + - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post" + - "TP=2 and TP=4, concurrency 4-256 for 1k1k and 8k1k sequence lengths" + - "Add --max-num-seqs and --gpu-memory-utilization 0.9 to server launch" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - glm5.1-fp4-mi355x-sglang + description: + - "Add GLM-5.1 MXFP4 single-node MI355X SGLang benchmark" + - "Image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260415" + - "TP=2 and TP=4 (limited concurrency), concurrency 4-256 for 1k1k and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 01eec06a94098f81552edb6831f1b2ad461be13e Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 23 Apr 2026 17:59:02 +0900 Subject: [PATCH 07/12] remove glm5.1 fp4 mi355x sglang perf-changelog entry Co-Authored-By: Claude Sonnet 4.6 --- perf-changelog.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ba69264d4..43069c648 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1726,11 +1726,3 @@ - "TP=2 and TP=4, concurrency 4-256 for 1k1k and 8k1k sequence lengths" - "Add --max-num-seqs and --gpu-memory-utilization 0.9 to server launch" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - -- config-keys: - - glm5.1-fp4-mi355x-sglang - description: - - "Add GLM-5.1 MXFP4 single-node MI355X SGLang benchmark" - - "Image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260415" - - "TP=2 and TP=4 (limited concurrency), concurrency 4-256 for 1k1k and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 7faa1d50c1b236748a59a0a9ac616f81ed857cc7 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 23 Apr 2026 17:59:29 +0900 Subject: [PATCH 08/12] update glm5.1 fp4 mi355x atom pr link Co-Authored-By: Claude Sonnet 4.6 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 43069c648..e3948c918 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1725,4 +1725,4 @@ - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post" - "TP=2 and TP=4, concurrency 4-256 for 1k1k and 8k1k sequence lengths" - "Add --max-num-seqs and --gpu-memory-utilization 0.9 to server launch" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1043 From 156d7cf3f9a2560821f8de1ae3b2f1b6581449ec Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 23 Apr 2026 18:00:22 +0900 Subject: [PATCH 09/12] fix whitespace in glm5.1 fp4 mi355x atom benchmark script Co-Authored-By: Claude Sonnet 4.6 --- benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh b/benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh index f1ff0207c..1d1b4329a 100644 --- a/benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh @@ -49,7 +49,7 @@ python3 -m atom.entrypoints.openai_server \ -tp $TP \ --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ --max-num-seqs $CONC \ - --gpu-memory-utilization $MEM_FRAC_STATIC\ + --gpu-memory-utilization $MEM_FRAC_STATIC \ --default-chat-template-kwargs '{"enable_thinking": false}' \ --trust-remote-code \ > $SERVER_LOG 2>&1 & From 7126346c19adcafe5e1f15d76fa745d9ab1934ba Mon Sep 17 00:00:00 2001 From: seungrokj <144636725+seungrokj@users.noreply.github.com> Date: Thu, 23 Apr 2026 17:23:09 +0800 Subject: [PATCH 10/12] Update model-prefix for glm5.1-fp4-mi355x-atom --- .github/configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 14b048558..326994fb5 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -360,7 +360,7 @@ glm5.1-fp4-mi355x-sglang: glm5.1-fp4-mi355x-atom: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post model: amd/GLM-5.1-MXFP4 - model-prefix: glm5 + model-prefix: glm5.1 runner: mi355x precision: fp4 framework: atom From c679105f6bde6c5b5f3852a51faa715c13f0078c Mon Sep 17 00:00:00 2001 From: seungrokj <144636725+seungrokj@users.noreply.github.com> Date: Thu, 23 Apr 2026 17:45:21 +0800 Subject: [PATCH 11/12] Update glm5.1_fp4_mi355x_atom.sh --- benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh b/benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh index 1d1b4329a..036346af3 100644 --- a/benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/glm5.1_fp4_mi355x_atom.sh @@ -48,7 +48,6 @@ python3 -m atom.entrypoints.openai_server \ --server-port $PORT \ -tp $TP \ --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --max-num-seqs $CONC \ --gpu-memory-utilization $MEM_FRAC_STATIC \ --default-chat-template-kwargs '{"enable_thinking": false}' \ --trust-remote-code \ From ba511782711c67bb4c80da0bc6aff3fa9416ad89 Mon Sep 17 00:00:00 2001 From: seungrokj <144636725+seungrokj@users.noreply.github.com> Date: Thu, 23 Apr 2026 19:13:29 +0800 Subject: [PATCH 12/12] Update amd-master.yaml --- .github/configs/amd-master.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 326994fb5..4e92b2b03 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -369,12 +369,10 @@ glm5.1-fp4-mi355x-atom: - isl: 1024 osl: 1024 search-space: - - { tp: 2, conc-start: 4, conc-end: 256 } - { tp: 4, conc-start: 4, conc-end: 256 } - isl: 8192 osl: 1024 search-space: - - { tp: 2, conc-start: 4, conc-end: 256 } - { tp: 4, conc-start: 4, conc-end: 256 } kimik2.5-int4-mi355x-vllm: