From d9557863eabcfed3bcc231bbf1325c450e1e1be4 Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Thu, 30 Apr 2026 16:00:27 +0800 Subject: [PATCH 01/11] Update amd-master.yaml --- .github/configs/amd-master.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 1faf3682b..cdd72aeaa 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -533,6 +533,31 @@ minimaxm2.5-fp8-mi355x-vllm: - { tp: 4, ep: 4, conc-start: 4, conc-end: 512 } - { tp: 8, ep: 8, conc-start: 2, conc-end: 2 } +minimaxm2.5-fp8-mi355x-vllm-eagle3: + image: vllm/vllm-openai-rocm:nightly-4eafc729285e459a5fc96efd6f7b313b155cad48 + model: MiniMaxAI/MiniMax-M2.5 + model-prefix: minimaxm2.5 + runner: mi355x + precision: fp8 + framework: vllm + multinode: false + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 4, ep: 4, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + - { tp: 8, ep: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + - isl: 1024 + osl: 8192 + search-space: + - { tp: 4, ep: 4, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + - { tp: 8, ep: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, ep: 4, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + - { tp: 8, ep: 8, conc-start: 4, conc-end: 64, spec-decoding: eagle3 } + minimaxm2.5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.1-ubuntu24.04-pytorch2.9.1-atom0.1.2 model: MiniMaxAI/MiniMax-M2.5 From b4da9def33c2789b508bcb45f08d94ec8e8115ea Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Thu, 30 Apr 2026 16:01:09 +0800 Subject: [PATCH 02/11] Update README.md --- .github/workflows/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index de0a3dcab..67d3d4dd5 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -235,7 +235,7 @@ The corresponding `SingleNodeMatrixEntry` enforces these same fields with approp 2. **`extra='forbid'`**: Unknown fields are rejected, preventing typos or deprecated fields from slipping through. -3. **Strict typing**: Fields like `spec-decoding` use `Literal["mtp", "draft_model", "none"]` to restrict values to known options. +3. **Strict typing**: Fields like `spec-decoding` use `Literal` types to restrict values to known options; see `utils/matrix_logic/validation.py` for the current set. 4. **Concurrency validation**: The system ensures either `conc-list` OR `conc-start`/`conc-end` is provided, but not both. From 9e95d1d99a522fdc70554ad9d88f00bee9931fed Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Thu, 30 Apr 2026 16:02:08 +0800 Subject: [PATCH 03/11] Create minimaxm2.5_fp8_mi355x_eagle3.sh --- .../minimaxm2.5_fp8_mi355x_eagle3.sh | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 benchmarks/single_node/minimaxm2.5_fp8_mi355x_eagle3.sh diff --git a/benchmarks/single_node/minimaxm2.5_fp8_mi355x_eagle3.sh b/benchmarks/single_node/minimaxm2.5_fp8_mi355x_eagle3.sh new file mode 100644 index 000000000..bef2732b2 --- /dev/null +++ b/benchmarks/single_node/minimaxm2.5_fp8_mi355x_eagle3.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + EP_SIZE \ + CONC \ + ISL \ + OSL \ + MAX_MODEL_LEN \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +hf download "$MODEL" + +SPEC_DRAFT_MODEL=${SPEC_DRAFT_MODEL:-MiniMaxAI/MiniMax-M2.5-Eagle3} +SPEC_NUM_TOKENS=${SPEC_NUM_TOKENS:-3} +SPEC_DRAFT_TP=${SPEC_DRAFT_TP:-1} + +hf download "$SPEC_DRAFT_MODEL" + +# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ +if [ -n "$ROCR_VISIBLE_DEVICES" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 + +SERVER_LOG=/workspace/server.log +PORT=${PORT:-8888} + +if [ "${EVAL_ONLY}" = "true" ]; then + setup_eval_context + MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi + +if [ "$EP_SIZE" -gt 1 ]; then + EP=" --enable-expert-parallel" +else + EP=" " +fi + +# Eagle3 speculative config. Double-quoted JSON string with escaped inner quotes +# so the draft model and token count expand before being passed as one argument. +SPEC_CONFIG="{\"model\":\"${SPEC_DRAFT_MODEL}\",\"method\":\"eagle3\",\"num_speculative_tokens\":${SPEC_NUM_TOKENS},\"draft_tensor_parallel_size\":${SPEC_DRAFT_TP}}" + +# Start GPU monitoring (power, temperature, clocks every second) +start_gpu_monitor + +set -x +vllm serve $MODEL --port $PORT \ +--tensor-parallel-size=$TP \ +$EP \ +--gpu-memory-utilization 0.95 \ +--max-model-len $MAX_MODEL_LEN \ +--kv-cache-dtype fp8 \ +--block-size=32 \ +--no-enable-prefix-caching \ +--attention-backend "ROCM_AITER_FA" \ +--reasoning-parser minimax_m2 \ +--trust-remote-code \ +--speculative-config "$SPEC_CONFIG" > $SERVER_LOG 2>&1 & + +SERVER_PID=$! + +# Wait for server to be ready +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +# EAGLE-style speculative decoding is trained on chat-formatted prompts. +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --use-chat-template \ + --trust-remote-code + +# After throughput, run evaluation only if RUN_EVAL is true +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +# Stop GPU monitoring +stop_gpu_monitor +set +x From 7ce71d10cba544e0ed54fe2a1dc8536aaa70bf5a Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Thu, 30 Apr 2026 16:02:49 +0800 Subject: [PATCH 04/11] Update launch_mi355x-amds.sh --- runners/launch_mi355x-amds.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 152745d4e..efc99639c 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -179,7 +179,11 @@ else export PORT_OFFSET=${RUNNER_NAME: -1} export PORT=$(( 8888 + ${PORT_OFFSET} )) FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "atom" ]] && printf '_atom' || printf '') - SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') + case "$SPEC_DECODING" in + mtp) SPEC_SUFFIX='_mtp' ;; + eagle3) SPEC_SUFFIX='_eagle3' ;; + *) SPEC_SUFFIX='' ;; + esac PARTITION="compute" SQUASH_FILE="/var/lib/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" From 348f70a637c03dd1efdcf01edec01014413e46c4 Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Thu, 30 Apr 2026 16:03:37 +0800 Subject: [PATCH 05/11] Update test_validation.py --- utils/matrix_logic/test_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 0f1f44c27..636875e39 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -275,7 +275,7 @@ def test_conc_as_list(self, valid_single_node_matrix_entry): def test_spec_decoding_values(self, valid_single_node_matrix_entry): """Spec decoding should accept valid literal values.""" - for value in ["mtp", "draft_model", "none"]: + for value in ["mtp", "draft_model", "eagle3", "none"]: valid_single_node_matrix_entry["spec-decoding"] = value entry = SingleNodeMatrixEntry(**valid_single_node_matrix_entry) assert entry.spec_decoding == value From 740cab069b4a3348e979141635b8ba26ab604a50 Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Thu, 30 Apr 2026 16:04:04 +0800 Subject: [PATCH 06/11] Update validation.py --- utils/matrix_logic/validation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index ce10840b5..bdf7a8494 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -77,7 +77,7 @@ class SingleNodeMatrixEntry(BaseModel): model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) precision: str framework: str - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "eagle3", "none"] = Field( alias=Fields.SPEC_DECODING.value ) runner: str @@ -116,7 +116,7 @@ class MultiNodeMatrixEntry(BaseModel): model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) precision: str framework: str - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "eagle3", "none"] = Field( alias=Fields.SPEC_DECODING.value ) runner: str @@ -204,7 +204,7 @@ class SingleNodeSearchSpaceEntry(BaseModel): tp: int ep: Optional[int] = None - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "eagle3", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value) dp_attn: Optional[bool] = Field( default=None, alias=Fields.DP_ATTN.value) @@ -224,7 +224,7 @@ class MultiNodeSearchSpaceEntry(BaseModel): """Multinode search space configuration.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "eagle3", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value) prefill: WorkerConfig decode: WorkerConfig From df3b1fea1352e59ea7ac80b65fa5da0c4fb8b389 Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Thu, 30 Apr 2026 16:04:33 +0800 Subject: [PATCH 07/11] Update perf-changelog.yaml --- perf-changelog.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ae2b88f03..903ae2cd7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -2031,3 +2031,14 @@ - "Topologies mirror the dsv4-fp4-gb300-dynamo-vllm sibling: low-conc 1p1d-dep8-tep8 (4 nodes), mid 1p1d-dep8-dep16 (6 nodes), high 3p1d-dep8-dep16 (10 nodes). 4096 overlap between mid and high gives a topology-crossover A/B" - "No upstream GB300 DSV4 sglang disagg recipe exists. Per-worker sglang_config (env vars + flashinfer_mxfp4 + chunked-prefill-size 4096 + disable-flashinfer-autotune + mem-fraction-static 0.82) is mirrored from NVIDIA/srt-slurm PR #69 (recipes/gb300-fp4/1k1k-dsv4/agg-2n-low-latency.yaml — GB300 DSV4 SGLang aggregated). Disagg flag set (nixl transfer backend, enable-dp-attention + moe-a2a-backend deepep) cross-checked against PR #75 (recipes/gb300-fp4/1k1k-dsv4/disagg-1p1d-tp4-mxfp4.yaml — GB300 DSV4 SGLang disagg) and the SGLang DeepSeek-V4 cookbook. Stored under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/ and overlaid onto the upstream srt-slurm checkout at runtime" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1157 + +- config-keys: + - minimaxm2.5-fp8-mi355x-vllm-eagle3 + description: + - "Add MiniMax-M2.5 FP8 vLLM Eagle3 speculative decoding benchmark for MI355X" + - "Model: MiniMaxAI/MiniMax-M2.5 from Hugging Face Hub" + - "Draft model: MiniMaxAI/MiniMax-M2.5-Eagle3 from Hugging Face Hub" + - "Recipe uses TP/EP={4/4, 8/8}, num_speculative_tokens=3, draft_tensor_parallel_size=1" + - "Sweep covers ISL/OSL 1k/1k, 1k/8k, and 8k/1k with concurrency 4-64" + - "Image pinned to vllm/vllm-openai-rocm:nightly-4eafc729285e459a5fc96efd6f7b313b155cad48 for Eagle3 support" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD From c871601213d1d431f56133ba687a8ebf3bdabd6b Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Thu, 30 Apr 2026 16:08:46 +0800 Subject: [PATCH 08/11] Update perf-changelog.yaml --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 903ae2cd7..1f86d6165 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -2041,4 +2041,4 @@ - "Recipe uses TP/EP={4/4, 8/8}, num_speculative_tokens=3, draft_tensor_parallel_size=1" - "Sweep covers ISL/OSL 1k/1k, 1k/8k, and 8k/1k with concurrency 4-64" - "Image pinned to vllm/vllm-openai-rocm:nightly-4eafc729285e459a5fc96efd6f7b313b155cad48 for Eagle3 support" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1234 From 140e38691ae9734804d91a0b9c99089411b311a6 Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Thu, 30 Apr 2026 16:24:09 +0800 Subject: [PATCH 09/11] Update perf-changelog.yaml --- perf-changelog.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1f86d6165..aa5c49bc6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -2036,9 +2036,9 @@ - minimaxm2.5-fp8-mi355x-vllm-eagle3 description: - "Add MiniMax-M2.5 FP8 vLLM Eagle3 speculative decoding benchmark for MI355X" - - "Model: MiniMaxAI/MiniMax-M2.5 from Hugging Face Hub" - - "Draft model: MiniMaxAI/MiniMax-M2.5-Eagle3 from Hugging Face Hub" + - "Image: vllm/vllm-openai-rocm:nightly-4eafc729285e459a5fc96efd6f7b313b155cad48" + - "Model: MiniMaxAI/MiniMax-M2.5" + - "Draft model: thoughtworks/MiniMax-M2.5-Eagle3 (num_speculative_tokens=3, draft_tp=1)" - "Recipe uses TP/EP={4/4, 8/8}, num_speculative_tokens=3, draft_tensor_parallel_size=1" - "Sweep covers ISL/OSL 1k/1k, 1k/8k, and 8k/1k with concurrency 4-64" - - "Image pinned to vllm/vllm-openai-rocm:nightly-4eafc729285e459a5fc96efd6f7b313b155cad48 for Eagle3 support" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1234 From ad0195923bf74e3899546720d5cdeb1f734df385 Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Thu, 30 Apr 2026 16:25:01 +0800 Subject: [PATCH 10/11] Update perf-changelog.yaml --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index aa5c49bc6..fb6ea8ede 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -2038,7 +2038,7 @@ - "Add MiniMax-M2.5 FP8 vLLM Eagle3 speculative decoding benchmark for MI355X" - "Image: vllm/vllm-openai-rocm:nightly-4eafc729285e459a5fc96efd6f7b313b155cad48" - "Model: MiniMaxAI/MiniMax-M2.5" - - "Draft model: thoughtworks/MiniMax-M2.5-Eagle3 (num_speculative_tokens=3, draft_tp=1)" + - "Draft model: thoughtworks/MiniMax-M2.5-Eagle3" - "Recipe uses TP/EP={4/4, 8/8}, num_speculative_tokens=3, draft_tensor_parallel_size=1" - "Sweep covers ISL/OSL 1k/1k, 1k/8k, and 8k/1k with concurrency 4-64" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1234 From 63cd9a387728f64129a6e5aa284c4bb9a02009bc Mon Sep 17 00:00:00 2001 From: larryli2-amd Date: Fri, 1 May 2026 14:33:00 +0800 Subject: [PATCH 11/11] Update minimaxm2.5_fp8_mi355x_eagle3.sh --- benchmarks/single_node/minimaxm2.5_fp8_mi355x_eagle3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/minimaxm2.5_fp8_mi355x_eagle3.sh b/benchmarks/single_node/minimaxm2.5_fp8_mi355x_eagle3.sh index bef2732b2..40d0fd18d 100644 --- a/benchmarks/single_node/minimaxm2.5_fp8_mi355x_eagle3.sh +++ b/benchmarks/single_node/minimaxm2.5_fp8_mi355x_eagle3.sh @@ -19,7 +19,7 @@ fi hf download "$MODEL" -SPEC_DRAFT_MODEL=${SPEC_DRAFT_MODEL:-MiniMaxAI/MiniMax-M2.5-Eagle3} +SPEC_DRAFT_MODEL=${SPEC_DRAFT_MODEL:-thoughtworks/MiniMax-M2.5-Eagle3} SPEC_NUM_TOKENS=${SPEC_NUM_TOKENS:-3} SPEC_DRAFT_TP=${SPEC_DRAFT_TP:-1}