From ba03630403b9a8c0efd37968abf34548e7573b12 Mon Sep 17 00:00:00 2001 From: tianhao909 <843101550@qq.com> Date: Tue, 14 Apr 2026 08:51:33 +0000 Subject: [PATCH 1/8] feat: SimAI 1.6 GPU memory module with PD-separation (review fixes) --- README.md | 124 +- README_CN.md | 268 +++++ vidur-alibabacloud/.gitignore | 17 + vidur-alibabacloud/README-vidur.md | 162 ++- vidur-alibabacloud/README.md | 340 ++++-- vidur-alibabacloud/README_CN.md | 458 ++++++++ .../hf_configs/deepseek_R1_0528_config.json | 67 ++ .../data/hf_configs/deepseek_v3_config.json | 70 ++ .../qwen3-235B-A22B_FP8_config.json | 49 + .../hf_configs/qwen3-235B-A22B_config.json | 38 + .../data/hf_configs/qwen3-30B-A3B_config.json | 38 + .../data/hf_configs/qwen3-8B_config.json | 30 + ...wen3-next-80B-A3B_Instruct_FP8_config.json | 43 + .../hf_configs/qwen3-next-80B-A3B_config.json | 43 + .../vidur-ali-scenarios/run_scenarios.sh | 361 ++++++ .../tests/test_pd_separation.py | 167 +++ vidur-alibabacloud/vidur/config/config.py | 111 +- .../vidur/config/device_sku_config.py | 38 +- .../vidur/config/model_config.py | 122 +- .../vidur/config/node_sku_config.py | 11 +- vidur-alibabacloud/vidur/entities/batch.py | 1 - vidur-alibabacloud/vidur/entities/cluster.py | 142 ++- .../vidur/entities/execution_time.py | 1027 +++++++++++++++-- vidur-alibabacloud/vidur/entities/replica.py | 182 ++- vidur-alibabacloud/vidur/entities/request.py | 156 +-- .../vidur/events/batch_end_event.py | 178 ++- .../vidur/events/batch_stage_arrival_event.py | 1 + .../vidur/events/batch_stage_end_event.py | 15 +- .../vidur/events/replica_schedule_event.py | 4 +- .../events/replica_stage_schedule_event.py | 9 +- .../base_execution_time_predictor.py | 113 +- .../communication_time_predictor.py | 30 +- .../sklearn_execution_time_predictor.py | 84 +- .../vidur/metrics/cdf_sketch.py | 4 +- .../vidur/metrics/data_series.py | 60 +- .../vidur/metrics/metrics_store.py | 3 +- .../profiling/collectives/benchmark_runner.py | 4 +- .../profiling/collectives/collectives_impl.py | 9 +- .../splitwise_global_scheduler.py | 63 +- .../splitwise_replica_scheduler.py | 141 ++- .../replica_stage_schduler.py | 7 +- .../vidur/scheduler/utils/memory_planner.py | 302 ++++- vidur-alibabacloud/vidur/simulator.py | 22 +- .../vidur/types/device_sku_type.py | 3 + .../vidur/types/node_sku_type.py | 1 + .../vidur/utils/mfu_calculator.py | 93 +- .../vidur/utils/param_counter.py | 457 +++++++- 47 files changed, 5056 insertions(+), 612 deletions(-) create mode 100644 README_CN.md create mode 100644 vidur-alibabacloud/README_CN.md create mode 100644 vidur-alibabacloud/data/hf_configs/deepseek_R1_0528_config.json create mode 100644 vidur-alibabacloud/data/hf_configs/deepseek_v3_config.json create mode 100644 vidur-alibabacloud/data/hf_configs/qwen3-235B-A22B_FP8_config.json create mode 100644 vidur-alibabacloud/data/hf_configs/qwen3-235B-A22B_config.json create mode 100644 vidur-alibabacloud/data/hf_configs/qwen3-30B-A3B_config.json create mode 100644 vidur-alibabacloud/data/hf_configs/qwen3-8B_config.json create mode 100644 vidur-alibabacloud/data/hf_configs/qwen3-next-80B-A3B_Instruct_FP8_config.json create mode 100644 vidur-alibabacloud/data/hf_configs/qwen3-next-80B-A3B_config.json create mode 100644 vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh create mode 100644 vidur-alibabacloud/tests/test_pd_separation.py diff --git a/README.md b/README.md index 85fbcb63..d2b2aba3 100755 --- a/README.md +++ b/README.md @@ -1,12 +1,29 @@ +

+ 中文  ｜  English +

+ +# SimAI + +[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) +[![NSDI'25](https://img.shields.io/badge/NSDI'25-SimAI-blue.svg)](https://ennanzhai.github.io/pub/nsdi25spring-simai.pdf) + # Latest News ### Recent Updates +- [2026/03] **SimAI 1.6 Released!** This release adds GPU memory modeling for inference simulation. Key features include: + + - **GPU Memory Module:** Accurate parameter counting and KV cache management for DeepSeek-V3-671B, Qwen3-MoE-235B, and Qwen3-Next-80B. + - **PD-Separation Memory Planning:** Independent parameter memory and KV cache budget calculation for Prefill and Decode phases. See [memory_planner.py](./vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py). + - **Improved Decode Time Estimation:** Linear interpolation replacing nearest-neighbor for AICB decode time prediction, with global cache for cross-run reuse. See [execution_time.py](./vidur-alibabacloud/vidur/entities/execution_time.py). + - **4-Scenario Test Suite:** End-to-end validation covering Qwen3-Next-80B, DeepSeek-671B, and Qwen3-MoE-235B. See [run_scenarios.sh](./vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh). + - **Code Quality:** Replaced print with logging, added bilingual docstrings, removed ~390 lines of dead code, standardized TODOs, and added type annotations across vidur-alibabacloud modules. + - [2025/12] **SimAI 1.5 Released!** This release brings end-to-end simulation for multi-request **inference** workloads. Key features include: - - - **Advanced Inference Simulation:** Model complex scenarios with Prefill/Decode separation. - - **Modern Model Support:** Now includes DeepSeek, Qwen3Moe and Qwen3Next. See [AICB's README](./aicb/README.md) for more detailed information. - - **Request Scheduling:** Request scheduling is now handled by a component adapted from Microsoft's [Vidur](https://github.com/microsoft/vidur). See [Vidur-Alibabacloud's README](./vidur-alibabacloud/README.md) for more detailed information. + + - **Advanced Inference Simulation:** Model complex scenarios with Prefill/Decode separation. + - **Modern Model Support:** Now includes DeepSeek, Qwen3Moe and Qwen3Next. See [AICB's README](./aicb/README.md) for more detailed information. + - **Request Scheduling:** Request scheduling is now handled by a component adapted from Microsoft's [Vidur](https://github.com/microsoft/vidur). See [Vidur-Alibabacloud's README](./vidur-alibabacloud/README.md) for more detailed information. - [2025/11] [AICB](https://github.com/aliyun/aicb/tree/master) now supports generating **prefill/decode** inference workloads for **DeepSeek**, **Qwen3-MoE** and **Qwen3-Next**. @@ -14,7 +31,8 @@ - [2025/06] The code of SimCCL is first released in the branch [SimCCL](https://github.com/aliyun/SimAI/tree/SimCCL) and will be released in SimCCL repository soon. -**We warmly welcome contributions from the community!** If you are interested in helping shape the future of SimAI, please feel free to open an issue to discuss your ideas or submit a pull request. +**We warmly welcome contributions from the community!** If you are interested in helping shape the future of SimAI, please feel free to open an issue to discuss your ideas or submit a pull request. +
🎯 Events & Community Engagement 🎯 @@ -29,6 +47,7 @@ | Date | Event | Location | Content | Type | |:----------------:|:------------------------------------------------------------------------ |:----------------------- |:-------------------------------------------------------- |:-------------:| +| Mar 16, 2026 | SimAI 1.6 | 🌐 Online | The release of SimAI 1.6 | 💻 Virtual | | Dec 30, 2025 | SimAI 1.5 | 🌐 Online | The release of SimAI 1.5 | 💻 Virtual | | Jun 4, 2025 | The first workshop of the SimAI community | 📍 Peking University | Three talks from community contributors | 🎓 On-site | | May 24, 2025 | The 28th Chinasys workshop | 📍 Chongqing University | An invited talk about SimAI | 🎓 On-site | @@ -44,6 +63,12 @@ --- +## Documentation + +See [Tutorial](./docs/Tutorial.md) for full documentation. + +--- + # Table of Contents - [SimAI Overview](#simai-overview) @@ -51,18 +76,17 @@ - [Components](#components) - [Scenario](#scenario) - [Citation](#citation) -- [Usage](#usage) +- [Quick Start](#quick-start) - [Setup](#setup) - - [From Source Code](#from-source-code) - [Use SimAI-Analytical](#use-simai-analytical) - [Use SimAI-Simulation](#use-simai-simulation) - - [Use Vidur-AICB](#use-vidur-aicb) + - [Use Multi-requests Inference Simulation](#use-multi-requests-inference-simulation) # SimAI Overview ## Introduction -**SimAI** is the industry's first full-stack, high-precision **Sim**ulator for **AI** large-scale **\*\*inference\*\*** and **training**. It provides detailed modeling and simulation of the entire LLM training process, encompassing framework, collective communication, network layers, and more. This comprehensive approach offers end-to-end performance data, enabling researchers to: +**SimAI** is the industry's first full-stack, high-precision **Sim**ulator for **AI** large-scale **inference** and **training**. It provides detailed modeling and simulation of the entire LLM training process, encompassing framework, collective communication, network layers, and more. This comprehensive approach offers end-to-end performance data, enabling researchers to: - Analyze inference/training process details - Evaluate the time consumption of AI tasks under specific conditions @@ -86,7 +110,7 @@ SimAI --|--- SimCCL |--- vidur-alibabacloud -Building on pure simulation capabilities, SimAI has evolved into a versatile full-stack toolkit comprising four components ([aicb](https://github.com/aliyun/aicb), [SimCCL](https://github.com/aliyun/SimCCL), [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud), [ns-3-alibabacloud](https://github.com/aliyun/ns-3-alibabacloud)). These components can be combined in various ways to achieve different functionalities. Below, we present the six main usage scenarios for SimAI. We encourage users to explore even more possibilities with this powerful tool. +Building on pure simulation capabilities, SimAI has evolved into a versatile full-stack toolkit comprising four components ([aicb](https://github.com/aliyun/aicb), [SimCCL](https://github.com/aliyun/SimCCL), [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud), [ns-3-alibabacloud](https://github.com/aliyun/ns-3-alibabacloud)). These components can be combined in various ways to achieve different functionalities. Below, we present the main usage scenarios for SimAI. We encourage users to explore even more possibilities with this powerful tool. Below is the architecture diagram of the SimAI Simulator: ![SimAI_Arc](./docs/images/SimAI_Arc.png) @@ -103,15 +127,15 @@ SimAI supports three major operation modes to meet different simulation requirem **SimAI-Physical** *(Beta)* enables physical traffic generation for CPU RDMA cluster environments. This mode generates NCCL-like traffic patterns, allowing in-depth study of NIC behaviors during LLM training. It is currently in internal testing phase. -| Scenario | Description | Component Combination | -| -------------------------------------- | ------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1. AICB Test Suite | Run communication patterns on GPU clusters using AICB Test suite | [AICB](https://github.com/aliyun/aicb) | -| 2. AICB/AIOB Workload | Model compute/communication patterns of **\*\*inference\*\*/training** process to generate workload | [AICB](https://github.com/aliyun/aicb) | -| 3. Collective Comm Analyze | Break down collective communication operations into point-to-point communication sets | [SimCCL](https://github.com/aliyun/SimCCL) | -| 4. Collective Comm w/o GPU | Perform RDMA collective communication traffic on non-GPU clusters | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(physical) | +| Scenario | Description | Component Combination | +|----------------------------------------|-----------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1. AICB Test Suite | Run communication patterns on GPU clusters using AICB Test suite | [AICB](https://github.com/aliyun/aicb) | +| 2. AICB/AIOB Workload | Model compute/communication patterns of **inference**/training process to generate workload | [AICB](https://github.com/aliyun/aicb) | +| 3. Collective Comm Analyze | Break down collective communication operations into point-to-point communication sets | [SimCCL](https://github.com/aliyun/SimCCL) | +| 4. Collective Comm w/o GPU | Perform RDMA collective communication traffic on non-GPU clusters | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(physical) | | 5. SimAI-Analytical | Conduct rapid AICB workload analysis and simulation on any server (ignoring underlying network details) | [AICB](https://github.com/aliyun/aicb) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(analytical) | -| 6. SimAI-Simulation | Perform full simulation on any server | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(simulation) + [ns-3-alibabacloud](https://github.com/aliyun/ns-3-alibabacloud) | -| 7. Multi-requests Inference Simulation | Perform full multi-requests **inference** simulation using one GPU server | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [vidur-alibabacloud](./vidur-alibabacloud) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(analytical/simulation) | +| 6. SimAI-Simulation | Perform full simulation on any server | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(simulation) + [ns-3-alibabacloud](https://github.com/aliyun/ns-3-alibabacloud) | +| 7. Multi-requests Inference Simulation | Perform full multi-requests **inference** simulation using one GPU server | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [vidur-alibabacloud](./vidur-alibabacloud) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(analytical/simulation) | ## Citation @@ -125,15 +149,15 @@ We encourage innovative research and extensions based on SimAI. Welcome to join # Quick Start -Here are some simple examples, SimAI full tutorials can be found here: [**SimAI@Tutorial**](./docs/Tutorial.md), [**aicb@Tutorial**](https://github.com/aliyun/aicb/blob/master/training/tutorial.md), [SimCCL@Tutorial], [ns-3-alibabacloud@Tutorial] +Here are some simple examples. SimAI full tutorials can be found here: [**SimAI@Tutorial**](./docs/Tutorial.md), [**aicb@Tutorial**](https://github.com/aliyun/aicb/blob/master/training/tutorial.md), [SimCCL@Tutorial], [ns-3-alibabacloud@Tutorial] ## Setup -You can follow the instrucitons below to quickly set up the environtments and run SimAI +You can follow the instructions below to quickly set up the environments and run SimAI. ### From Source Code -The following code has been successfully tested on GCC/G++ 9.4.0, python 3.8.10 in Ubuntu 20.04 +The following code has been successfully tested on GCC/G++ 9.4.0, python 3.8.10 in Ubuntu 20.04. You can use the official Ubuntu 20.04 image, and do not install ninja. @@ -159,13 +183,13 @@ $ ./scripts/build.sh -c ns3 ## Use SimAI-Analytical ```bash -$ ./bin/SimAI_analytical -w example/workload_analytical.txt -g 9216 -g_p_s 8 -r test- -busbw example/busbw.yaml +$ ./bin/SimAI_analytical -w example/workload_analytical.txt -g 9216 -g_p_s 8 -r test- -busbw example/busbw.yaml ``` -For calculating bus bandwidth autolly, please try the following command: +For calculating bus bandwidth automatically, please try the following command: ```bash -$ ./bin/SimAI_analytical -w ./example/workload_analytical.txt -g 9216 -nv 360 -nic 48.5 -n_p_s 8 -g_p_s 8 -r example- +$ ./bin/SimAI_analytical -w ./example/workload_analytical.txt -g 9216 -nv 360 -nic 48.5 -n_p_s 8 -g_p_s 8 -r example- ``` ## Use SimAI-Simulation @@ -180,39 +204,57 @@ $ AS_SEND_LAT=3 AS_NVLS_ENABLE=1 ./bin/SimAI_simulator -t 16 -w ./example/microA ## Use Multi-requests Inference Simulation -For detailed information, please refer to the [README](./vidur-alibabacloud/README.md) file in the `vidur-alibabacloud` directory. This module leverages AICB to profile the computation time of **inference** workloads. Due to its reliance on specific hardware-accelerated libraries like DeepGEMM and FlashMLA, it is exclusively compatible with NVIDIA GPUs based on the **Hopper (SM90)** and **Blackwell (SM100)** architectures. +For detailed information, please refer to the [README](./vidur-alibabacloud/README.md) file in the `vidur-alibabacloud` directory. This module leverages AICB to profile the computation time of **inference** workloads. Due to its reliance on specific hardware-accelerated libraries like DeepGEMM and FlashMLA, it is exclusively compatible with NVIDIA GPUs based on the **Hopper (SM90)** and **Blackwell (SM100)** architectures. -```shell +```bash # Build from Dockerfile docker build -t image:latest . -docker run --gpus all -it --rm image:latest +docker run --gpus all -it --rm image:latest ``` -**Note**: please add `ENV FLASH_MLA_DISABLE_SM100=1` to Dockerfile if using Hopper GPUs. +**Note:** Please add `ENV FLASH_MLA_DISABLE_SM100=1` to Dockerfile if using Hopper GPUs. -# Acknowledgments +To quickly validate all supported inference scenarios (Qwen3-Next-80B, DeepSeek-671B, Qwen3-MoE-235B), use the bundled 4-scenario test suite: -A huge thanks to the following people and organizations who have contributed to this project: +```bash +# Prerequisites: conda activate vidur +bash vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh --all +# Or run a single scenario: +bash vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh --scenario 1 +``` -- TianHao Fu (Peking University) and [TELOS-syslab](https://github.com/TELOS-syslab/), +> **Prerequisites:** Requires `conda activate vidur` environment. See [Environment Setup](./vidur-alibabacloud/README.md#-environment-setup) for details. +> +> For detailed scenario configuration table and output file descriptions, see [Vidur-AlibabaCloud README](./vidur-alibabacloud/README.md#4-scenario-configuration). -- Parth Parikh (KEYSIGHT), +# Acknowledgments -- Sarah-Michelle Hammer & Ziyi Wang (TU-Berlin), +A huge thanks to the following people and organizations who have contributed to this project: -- Xinyue Li (BUPT), +- TianHao Fu (Peking University) and [TELOS-syslab](https://github.com/TELOS-syslab/) +- Parth Parikh (KEYSIGHT) +- Sarah-Michelle Hammer & Ziyi Wang (TU-Berlin) +- Xinyue Li (BUPT) +- Tong Chen (Zhejiang University) +- Ming Wang (BUPT) +- Tao Jiang (Institute of Computing Technology, Chinese Academy of Sciences) -- Tong Chen (Zhejiang University), +...and many other individual contributors from the community (See the [Contributors to aliyun/SimAI](https://github.com/aliyun/SimAI/graphs/contributors)). -- Ming Wang (BUPT), +We also thank Chenning Li (MIT CSAIL) who initiated the cooperation on integrating SimAI into [M4](https://github.com/netiken/m4), a new, innovative simulator. -- Tao Jiang (Institute of Computing Technology, Chinese Academy of Sciences), +**This project still welcomes more contributions and suggestions.** -and many other individual contributors from the community (See the [Contributors to aliyun/SimAI · GitHub](https://github.com/aliyun/SimAI/graphs/contributors)). +# Contributing -We also thank Chenning Li (MIT CSAIL) who initiated the cooperation on integrating SimAI into [M4](https://github.com/netiken/m4), a new, innovative simulator. +We welcome all contributions! Please read the following guides before getting started: -**This project still welcomes more contributions and suggestions**. +| | | +|---|---| +| [Contributing Guide](./CONTRIBUTING.md) | How to submit issues and pull requests | +| [Security Policy](./SECURITY.md) | How to report security vulnerabilities | +| [Code of Conduct](./CODE_OF_CONDUCT.md) | Our community standards | +| [Changelog](./CHANGELOG.md) | Version history from v1.5 onwards | # Contact us @@ -224,5 +266,3 @@ Welcome to join the SimAI community chat groups, with the DingTalk group on the SimAI DingTalk SimAI WeChat
- -
diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 00000000..0baf73a3 --- /dev/null +++ b/README_CN.md @@ -0,0 +1,268 @@ +

+ 中文  ｜  English +

+ +# SimAI + +[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) +[![NSDI'25](https://img.shields.io/badge/NSDI'25-SimAI-blue.svg)](https://ennanzhai.github.io/pub/nsdi25spring-simai.pdf) + +# 最新动态 + +### 近期更新 + +- [2026/03] **SimAI 1.6 正式发布！** 本版本新增推理仿真的 GPU 内存建模能力。主要特性包括： + + - **GPU 内存计算模块：** 支持 DeepSeek-V3-671B、Qwen3-MoE-235B、Qwen3-Next-80B 的精确参数计数与 KV Cache 管理。 + - **PD 分离内存规划：** Prefill 与 Decode 阶段独立的参数内存和 KV Cache 预算计算。详见 [memory_planner.py](./vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py)。 + - **Decode 时间估算改进：** 首尾线性插值替代最近邻的 AICB decode 时间预测，全局缓存支持跨运行复用。详见 [execution_time.py](./vidur-alibabacloud/vidur/entities/execution_time.py)。 + - **4 场景端到端测试：** 覆盖 Qwen3-Next-80B、DeepSeek-671B、Qwen3-MoE-235B 的完整验证套件。详见 [run_scenarios.sh](./vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh)。 + - **代码质量提升：** logging 替换 print 输出、双语 docstring、清理 ~390 行死代码、TODO 规范化、类型标注补全。 + +- [2025/12] **SimAI 1.5 正式发布！** 本版本新增对多请求**推理**工作负载的端到端仿真支持，主要特性包括： + + - **高级推理仿真：** 支持 Prefill/Decode 分离等复杂场景建模。 + - **主流模型支持：** 新增 DeepSeek、Qwen3Moe 和 Qwen3Next 模型。详见 [AICB README](./aicb/README.md)。 + - **请求调度：** 请求调度组件基于微软 [Vidur](https://github.com/microsoft/vidur) 适配，详见 [Vidur-Alibabacloud README](./vidur-alibabacloud/README_CN.md)。 + +- [2025/11] [AICB](https://github.com/aliyun/aicb/tree/master) 新增对 **DeepSeek**、**Qwen3-MoE** 和 **Qwen3-Next** 的 **prefill/decode** 推理工作负载生成支持。 + +- [2025/09] [AICB](https://github.com/aliyun/aicb/tree/master) 新增 DeepSeek 训练工作负载生成支持。感谢 [@parthpower](https://github.com/parthpower) 的贡献。 + +- [2025/06] SimCCL 代码首次在 [SimCCL](https://github.com/aliyun/SimAI/tree/SimCCL) 分支发布，后续将在独立仓库正式开源。 + +**欢迎社区贡献！** 如有想法，欢迎提交 Issue 讨论或发起 Pull Request。 + +
+🎯 活动与社区 🎯 + +### 📅 即将举办 + +| 日期 | 活动 | 地点 | 内容 | 形式 | +|:----:|:----- |:-------- |:------- |:----:| +| -- | | | | | + +### 🌟 往期活动 + +| 日期 | 活动 | 地点 | 内容 | 形式 | +|:----------------:|:------------------------------------------------------------------------ |:----------------------- |:-------------------------------------------------------- |:-------------:| +| Mar 16, 2026 | SimAI 1.6 | 🌐 线上 | SimAI 1.6 正式发布 | 💻 线上直播 | +| Dec 30, 2025 | SimAI 1.5 | 🌐 线上 | SimAI 1.5 正式发布 | 💻 线上直播 | +| Jun 4, 2025 | SimAI 社区第一届研讨会 | 📍 北京大学 | 三场社区贡献者演讲 | 🎓 线下 | +| May 24, 2025 | 第 28 届 Chinasys 研讨会 | 📍 重庆大学 | SimAI 受邀演讲 | 🎓 线下 | +| Dec 27, 2024 | SimAI 技术分享 | 📍 北京航空航天大学 | SimAI 技术分享与交流 | 🎓 线下 | +| Dec 6, 2024 | 香港科技大学技术研讨会 | 📍 香港科技大学（广州） | SimAI 技术分享与交流 | 🎓 线下 | +| Dec 5, 2024 | [Bench'24 会议](https://mp.weixin.qq.com/s/STic_E12xMhZRxhzK9wRnw) | 📍 广州 | SimAI 教程与深度技术专场 | 🎓 线下 | +| Nov 26, 2024 | SimAI 社区直播 | 🌐 线上 | 互动技术交流与演示（400+ 参与者） | 💻 线上直播 | +| Nov 15, 2024 | 技术研讨会 | 📍 千岛湖 | SimAI 线下技术交流 | 🎯 线下 | +| Oct 18, 2024 | 嘉宾讲座 | 📍 复旦大学 | SimAI 教程与公开课 | 🎓 线下 | +| Sept 24-26, 2024 | CCF HPC China 2024 | 📍 武汉 | SimAI 介绍与技术报告 | 🎤 会议 | + +
+ +--- + +## 文档 + +详见 [Tutorial](./docs/Tutorial.md) 获取完整文档。 + +--- + +# 目录 + +- [SimAI 概述](#simai-概述) + - [简介](#简介) + - [组件](#组件) + - [应用场景](#应用场景) + - [引用](#引用) +- [快速开始](#快速开始) + - [环境搭建](#环境搭建) + - [使用 SimAI-Analytical](#使用-simai-analytical) + - [使用 SimAI-Simulation](#使用-simai-simulation) + - [使用多请求推理仿真](#使用多请求推理仿真) + +# SimAI 概述 + +## 简介 + +**SimAI** 是业界首个全栈高精度 AI 大规模**推理**与**训练**模拟器（**Sim**ulator for **AI**）。它对 LLM 训练全流程进行详细建模和仿真，涵盖框架、集合通信、网络层等，提供端到端的性能数据，帮助研究人员： + +- 分析推理/训练过程细节 +- 评估特定条件下 AI 任务的耗时 +- 评估各类算法优化带来的 E2E 性能收益，包括： + - 框架参数配置 + - 集合通信算法 + - NCCL 环境变量 + - 网络传输协议 + - 拥塞控制算法 + - 自适应路由算法 + - 扩展/集合网络拓扑调整 + - …… + +## 组件 + +
+        |--- AICB
+SimAI --|--- SimCCL
+        |--- astra-sim-alibabacloud
+        |--- ns-3-alibabacloud
+        |--- vidur-alibabacloud
+
+ +在纯仿真能力基础上，SimAI 已演进为一个由四个组件（[aicb](https://github.com/aliyun/aicb)、[SimCCL](https://github.com/aliyun/SimCCL)、[astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)、[ns-3-alibabacloud](https://github.com/aliyun/ns-3-alibabacloud)）构成的全栈工具套件。这些组件可以灵活组合以实现不同功能。我们鼓励用户探索更多可能性。 + +下图为 SimAI 模拟器架构图： +![SimAI_Arc](./docs/images/SimAI_Arc.png) + +astra-sim-alibabacloud 基于 [astra-sim](https://github.com/astra-sim/astra-sim/tree/ASTRA-sim-1.0) 扩展开发。感谢 astra-sim 团队的优秀工作和开源贡献。我们在其基础上集成了 NCCL 算法并添加了若干新特性。 + +## 应用场景 + +SimAI 支持三种主要运行模式： + +**SimAI-Analytical** 通过使用总线带宽（busbw）抽象网络通信细节来估算集合通信时间，实现快速仿真。目前支持用户自定义 busbw，自动计算 busbw 功能即将推出。 + +**SimAI-Simulation** 提供基于细粒度网络通信建模的全栈仿真。利用 NS-3 或其他网络模拟器（当前 NS-3 已开源）实现对所有通信行为的详细仿真，力求高保真还原真实训练环境。 + +**SimAI-Physical** *(Beta)* 支持在 CPU RDMA 集群环境下生成物理流量，通过生成类 NCCL 的流量模式深入研究 LLM 训练中的 NIC 行为。当前处于内测阶段。 + +| 场景 | 描述 | 组件组合 | +|------|------|----------| +| 1. AICB 测试套件 | 在 GPU 集群上使用 AICB 测试套件运行通信模式 | [AICB](https://github.com/aliyun/aicb) | +| 2. AICB/AIOB 工作负载 | 建模**推理**/训练过程的计算/通信模式以生成工作负载 | [AICB](https://github.com/aliyun/aicb) | +| 3. 集合通信分析 | 将集合通信操作分解为点对点通信集合 | [SimCCL](https://github.com/aliyun/SimCCL) | +| 4. 无 GPU 集合通信 | 在非 GPU 集群上执行 RDMA 集合通信流量 | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(physical) | +| 5. SimAI-Analytical | 在任意服务器上快速进行 AICB 工作负载分析与仿真（忽略底层网络细节） | [AICB](https://github.com/aliyun/aicb) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(analytical) | +| 6. SimAI-Simulation | 在任意服务器上进行全栈仿真 | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(simulation) + [ns-3-alibabacloud](https://github.com/aliyun/ns-3-alibabacloud) | +| 7. 多请求推理仿真 | 在单 GPU 服务器上进行多请求**推理**全栈仿真 | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [vidur-alibabacloud](./vidur-alibabacloud) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(analytical/simulation) | + +## 引用 + +SimAI 论文已被 NSDI'25 Spring 接收，详情请参阅： + +*SimAI: Unifying Architecture Design and Performance Tuning for Large-Scale Large Language Model Training with Scalability and Precision.* + +[[pdf](https://ennanzhai.github.io/pub/nsdi25spring-simai.pdf)] / [[slides](./docs/SimAI_Intro_Online.pdf)] / [[video](https://n.dingtalk.com/dingding/live-room/index.html?roomId=OF5BkBUXVxmgsK7x&liveUuid=305736cd-aa70-498b-8003-2b471a53decd)] + +欢迎基于 SimAI 开展创新研究和功能扩展。欢迎加入社区群或通过邮件联系我们交流，我们可提供技术支持。 + +# 快速开始 + +以下为简单示例。完整教程请参见：[**SimAI@Tutorial**](./docs/Tutorial.md)、[**aicb@Tutorial**](https://github.com/aliyun/aicb/blob/master/training/tutorial.md)、[SimCCL@Tutorial]、[ns-3-alibabacloud@Tutorial] + +## 环境搭建 + +请按照以下步骤快速搭建环境并运行 SimAI。 + +### 从源码安装 + +以下步骤已在 Ubuntu 20.04 的 GCC/G++ 9.4.0、python 3.8.10 环境下验证。 + +可使用官方 Ubuntu 20.04 镜像，**不要安装 ninja**。 + +（对于工作负载生成场景，推荐直接使用 NGC 容器镜像。） + +```bash +# 克隆仓库 +$ git clone https://github.com/aliyun/SimAI.git +$ cd ./SimAI/ + +# 初始化子模块 +$ git submodule update --init --recursive +# 更新到最新提交 +$ git submodule update --remote + +# 编译 SimAI-Analytical +$ ./scripts/build.sh -c analytical + +# 编译 SimAI-Simulation (ns3) +$ ./scripts/build.sh -c ns3 +``` + +## 使用 SimAI-Analytical + +```bash +$ ./bin/SimAI_analytical -w example/workload_analytical.txt -g 9216 -g_p_s 8 -r test- -busbw example/busbw.yaml +``` + +若需自动计算总线带宽，请尝试： + +```bash +$ ./bin/SimAI_analytical -w ./example/workload_analytical.txt -g 9216 -nv 360 -nic 48.5 -n_p_s 8 -g_p_s 8 -r example- +``` + +## 使用 SimAI-Simulation + +```bash +# 生成网络拓扑 +$ python3 ./astra-sim-alibabacloud/inputs/topo/gen_Topo_Template.py -topo Spectrum-X -g 128 -gt A100 -bw 100Gbps -nvbw 2400Gbps + +# 运行仿真 +$ AS_SEND_LAT=3 AS_NVLS_ENABLE=1 ./bin/SimAI_simulator -t 16 -w ./example/microAllReduce.txt -n ./Spectrum-X_128g_8gps_100Gbps_A100 -c astra-sim-alibabacloud/inputs/config/SimAI.conf +``` + +## 使用多请求推理仿真 + +详情请参见 `vidur-alibabacloud` 目录下的 [README](./vidur-alibabacloud/README_CN.md)。该模块利用 AICB 对**推理**工作负载的计算时间进行 profiling。由于依赖 DeepGEMM 和 FlashMLA 等特定硬件加速库，目前仅兼容基于 **Hopper（SM90）** 和 **Blackwell（SM100）** 架构的 NVIDIA GPU。 + +```bash +# 从 Dockerfile 构建 +docker build -t image:latest . +docker run --gpus all -it --rm image:latest +``` + +**注意：** 若使用 Hopper GPU，请在 Dockerfile 中添加 `ENV FLASH_MLA_DISABLE_SM100=1`。 + +如需快速验证所有支持的推理场景（Qwen3-Next-80B、DeepSeek-671B、Qwen3-MoE-235B），可使用内置的四场景测试套件： + +```bash +# 前置条件：conda activate vidur +bash vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh --all +# 或单独运行某个场景： +bash vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh --scenario 1 +``` + +> **前置条件：** 需先激活 `conda activate vidur` 环境。详见 [环境配置](./vidur-alibabacloud/README_CN.md#-环境配置)。 +> +> 完整场景配置表与输出文件说明请参见 [Vidur-AlibabaCloud README](./vidur-alibabacloud/README_CN.md#四场景配置说明)。 + +# 致谢 + +衷心感谢以下人员和机构对本项目的贡献： + + +- TianHao Fu (Peking University) and [TELOS-syslab](https://github.com/TELOS-syslab/) +- Parth Parikh (KEYSIGHT) +- Sarah-Michelle Hammer & Ziyi Wang (TU-Berlin) +- Xinyue Li (BUPT) +- Tong Chen (Zhejiang University) +- Ming Wang (BUPT) +- Tao Jiang (Institute of Computing Technology, Chinese Academy of Sciences) + +……以及众多来自社区的个人贡献者（详见 [Contributors to aliyun/SimAI](https://github.com/aliyun/SimAI/graphs/contributors)）。 + +同时感谢 Chenning Li（MIT CSAIL）发起了将 SimAI 集成到 [M4](https://github.com/netiken/m4) 的合作——M4 是一个新型创新模拟器。 + +**本项目持续欢迎更多贡献与建议。** + +# 贡献指南 + +欢迎参与贡献！开始前请阅读以下指引： + +| | | +|---|---| +| [贡献指南](./CONTRIBUTING.zh-CN.md) | 如何提交 Issue 和 Pull Request | +| [安全政策](./SECURITY_CN.md) | 如何报告安全漏洞 | +| [行为准则](./CODE_OF_CONDUCT_CN.md) | 社区行为规范 | +| [更新日志](./CHANGELOG_CN.md) | v1.5 起的版本历史 | + +# 联系我们 + +如有任何问题，欢迎发送邮件至：Gang Lu（yunding.lg@alibaba-inc.com）、Feiyang Xue（xuefeiyang.xfy@alibaba-inc.com）或 Qingxu Li（qingxu.lqx@alibaba-inc.com）。 + +欢迎加入 SimAI 社区交流群，左侧为钉钉群，右侧为微信群。 + +
+ SimAI 钉钉群 + SimAI 微信群 +
diff --git a/vidur-alibabacloud/.gitignore b/vidur-alibabacloud/.gitignore index 5e0a4d9a..7efc924e 100644 --- a/vidur-alibabacloud/.gitignore +++ b/vidur-alibabacloud/.gitignore @@ -182,14 +182,31 @@ env_3 vidur.egg-info example +# > AICB workload data files (excluded from PR to keep repo lightweight) +data/aicb_workload/ + # > fth_test/ fth_run/ +fth-test-*/logs/ +fth-test-*/simulator_output/ +fth-test-*/.claude/ +examples/vidur-ali-scenarios/logs/ +examples/vidur-ali-scenarios/simulator_output/ SimAI/ core.* *.csv +!data/aicb_workload/*.csv +!data/aicb_workload/cache/ +!data/aicb_workload/cache/*.csv +!data/aicb_workload/cache/*.json # Added rule to ignore all *fth.py files *fth.py # Added rule to recursively ignore all *fth.py files in all directories **/*fth.py +# Personal dev notes / fth files +README-fth.md +**/README-fth.md +fth.sh +**/fth.sh diff --git a/vidur-alibabacloud/README-vidur.md b/vidur-alibabacloud/README-vidur.md index 6f1cd2d0..00c08c04 100644 --- a/vidur-alibabacloud/README-vidur.md +++ b/vidur-alibabacloud/README-vidur.md @@ -164,4 +164,164 @@ This project may contain trademarks or logos for projects, products, or services trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. -Any use of third-party trademarks or logos are subject to those third-party's policies. \ No newline at end of file +Any use of third-party trademarks or logos are subject to those third-party's policies. + +## SimAI / AICB 场景示例（一键运行） + +> 以下命令均在 `vidur-alibabacloud/` 目录下执行，需提前激活 `vidur` conda 环境。 +> 使用 AICB 后端 (`--random_forrest_execution_time_predictor_config_backend aicb`)， +> 设备为 H20 DGX (`h20_dgx`)，请求生成为 Poisson QPS=100，固定长度 prefill=100/decode=8。 +> 所有输入输出文件统一汇聚至 `examples/vidur-ali-scenarios/` 目录： +> - 脚本: `examples/vidur-ali-scenarios/run_scenarios.sh` +> - 运行日志: `examples/vidur-ali-scenarios/logs/scenario__.log` +> - 模拟输出: `examples/vidur-ali-scenarios/simulator_output//` + +### 场景汇总 + +| 场景 | 模型 | ws | TP | PP | EP | PD分离 | 调度器 | +|------|------|----|----|----|-----|--------|--------| +| 1 | Qwen3-Next-80B | 32 | 1 | 1 | 32 | 否 | lor | +| 2 | Qwen3-Next-80B | 8 (P=2,D=6) | 1 | 1 | auto | 是 | split_wise | +| 3 | DeepSeek-671B | 8 (P=2,D=6) | 8 | 1 | 8 | 是 | split_wise | +| 4 | Qwen3-MoE-235B | 8 (P=2,D=6) | 4 | 1 | 4 | 是 | split_wise | + +### 使用方法 + +```sh +# 激活环境 +conda activate vidur + +# 运行单个场景（1~4） +bash examples/vidur-ali-scenarios/run_scenarios.sh --scenario 1 + +# 顺序运行所有场景 +bash examples/vidur-ali-scenarios/run_scenarios.sh --all + +# 查看帮助 +bash examples/vidur-ali-scenarios/run_scenarios.sh --help +``` + +### 分场景命令（手动方式） + +**场景 1: Qwen3-Next-80B 无PD分离 (ws=32, lor)** + +```sh +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 32 \ + --replica_config_pd_node_ratio 1 \ + --global_scheduler_config_type lor \ + --replica_scheduler_config_type sarathi \ + --replica_config_model_name qwen3-next-80B \ + --replica_config_tensor_parallel_size 1 \ + --replica_config_num_pipeline_stages 1 +``` + +**场景 2: Qwen3-Next-80B PD分离 (P=2, D=6, split_wise)** + +```sh +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --replica_config_num_prefill_replicas 2 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name qwen3-next-80B \ + --replica_config_tensor_parallel_size 1 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_prefill_tensor_parallel_size 1 \ + --replica_config_prefill_num_pipeline_stages 1 \ + --replica_config_decode_tensor_parallel_size 1 \ + --replica_config_decode_num_pipeline_stages 1 +``` + +**场景 3: DeepSeek-671B PD分离 (tp=8, ep=8, split_wise)** + +```sh +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name deepseek-671B \ + --replica_config_tensor_parallel_size 8 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 8 +``` + +**场景 4: Qwen3-MoE-235B PD分离 (tp=4, ep=4, split_wise)** + +```sh +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name qwen3-moe-235B \ + --replica_config_tensor_parallel_size 4 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 4 +``` \ No newline at end of file diff --git a/vidur-alibabacloud/README.md b/vidur-alibabacloud/README.md index 244a1e47..2d6e5fe5 100644 --- a/vidur-alibabacloud/README.md +++ b/vidur-alibabacloud/README.md @@ -1,64 +1,125 @@ -# README +

+ 中文  ｜  English +

+# Vidur-AlibabaCloud -Vidur ([original](https://github.com/microsoft/vidur)) is a simulation framework for large language model (LLM) inference systems. -**Vidur-AlibabaCloud** (this repository) is a customized version optimized for Alibaba Cloud **SimAI** scenarios. It supports advanced features such as **Prefill–Decode (PD) disaggregation** and includes dedicated adaptations for state-of-the-art (SOTA) LLM models including **DeepSeek-V3-671B**, **Qwen3-MoE-235B**, **Qwen3-Next-80B**, and other models. +[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/) +[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) + +Vidur ([original](https://github.com/microsoft/vidur)) is a simulation framework for large language model (LLM) inference systems. +**Vidur-AlibabaCloud** (this repository) is a customized version optimized for Alibaba Cloud **SimAI** scenarios. It supports advanced features such as **Prefill–Decode (PD) disaggregation** and includes dedicated adaptations for SOTA LLM models including **DeepSeek-V3-671B**, **Qwen3-MoE-235B**, **Qwen3-Next-80B**, and others. + + +--- + +## Table of Contents + +- [Key Features](#key-features) +- [GPU Memory Calculation](#gpu-memory-calculation) +- [Supported Models](#supported-models) +- [Environment Setup](#-environment-setup) +- [Running Examples](#%EF%B8%8F-running-examples) + - [4-Scenario Configuration](#4-scenario-configuration) + - [Output Files](#output-files) +- [Key Input Parameters](#-key-input-parameter-reference) +- [Key Output Interpretation](#-key-output-interpretation) +- [Known Issues](#%EF%B8%8F-known-issues) +- [Help](#-help) --- ## Key Features -+ **Prefill–Decode (PD) Separation** – Enables running the prefill and decode stages on different nodes, allowing elastic resource allocation and performance isolation. -(Inspired by [splitwise-sim](https://github.com/Mutinifni/splitwise-sim)). -+ **Flexible Parallelism** – Supports: - - **Data Parallel (DP)** - - **Tensor Parallel (TP)** - - **Pipeline Parallel (PP)** - - **Expert Parallel (EP)** (support in progress) -Works for both **dense** and **Mixture-of-Experts (MoE)** models (MoE support in progress). -+ **Multiple Execution-Time Prediction Backends** – Choose from: - - **AICB/AIOB** - Partially supports computation kernels and TP, DP, PP, EP communication size for DeepSeek-V3-671B, Qwen3-Moe-235B, Qwen3-Next-80B - - **SimAi_simulation** – SimAI NS-3-based network simulation (supports TP) - - **SimAi_analytical** – SimAI analytical performance model (supports TP) - - **Native Vidur [original]** – Supports TP, DP, PP -+ **Workload Generation & Replay** – Replay real-world traces or generate synthetic requests using fixed or Poisson distributions. -+ **Fine-Grained Metrics** – Records: - - TTFT – Time to First Token - - TBT / TPOT – Time Between Tokens / Time Per Output Token - - End-to-end latency - - Communication cost - - Computation cost - - Scheduling delay + +- **Prefill–Decode (PD) Separation** — Enables running the prefill and decode stages on different nodes, allowing elastic resource allocation and performance isolation. + (Inspired by [splitwise-sim](https://github.com/Mutinifni/splitwise-sim)) +- **Flexible Parallelism** — Supports: + - **Data Parallel (DP)** + - **Tensor Parallel (TP)** + - **Pipeline Parallel (PP)** + - **Expert Parallel (EP)** (support in progress) + + Works for both **dense** and **Mixture-of-Experts (MoE)** models (MoE support in progress). +- **Multiple Execution-Time Prediction Backends** — Choose from: + - **AICB/AIOB** — Partially supports computation kernels and TP, DP, PP, EP communication size for DeepSeek-V3-671B, Qwen3-MoE-235B, Qwen3-Next-80B + - **SimAI Simulation** — SimAI NS-3-based network simulation (supports TP) + - **SimAI Analytical** — SimAI analytical performance model (supports TP) + - **Native Vidur [original]** — Supports TP, DP, PP +- **Workload Generation & Replay** — Replay real-world traces or generate synthetic requests using fixed or Poisson distributions. +- **Fine-Grained Metrics** — Records: + - TTFT — Time to First Token + - TBT / TPOT — Time Between Tokens / Time Per Output Token + - End-to-end latency + - Communication cost + - Computation cost + - Scheduling delay + +--- + +## GPU Memory Calculation + +This module provides accurate GPU memory estimation for modern MoE (Mixture-of-Experts) models during inference simulation, covering **model parameter memory**, **KV cache memory**, and **maximum batch size** calculation under Prefill–Decode (PD) disaggregation. + +### Supported Attention Architectures + +| Architecture | Model | Description | +|---|---|---| +| **MLA** (Multi-head Latent Attention) | DeepSeek-V3-671B | Uses LoRA-compressed KV cache (`kv_lora_rank` + `qk_rope_head_dim`) for reduced memory footprint | +| **MHA / GQA** (Multi-Head / Grouped-Query Attention) | Qwen3-MoE-235B | Standard KV cache with `num_kv_heads * head_dim` per token per layer | +| **Hybrid Full + Linear Attention** | Qwen3-Next-80B | Alternates between full attention and linear (GDN) attention every 4 layers | + +### Key Components + +- **`ParamCounter`** (`vidur/utils/param_counter.py`) — Computes per-layer and per-device parameter counts for MLA, MHA/GQA, linear attention, and MoE expert weights, with FP8 quantization support. Under PD disaggregation, it returns separate `(total_params, prefill_params, decode_params)` based on `prefill_world_size` / `decode_world_size`. +- **`MemoryPlanner`** (`vidur/scheduler/utils/memory_planner.py`) — Plans GPU memory budget: `available = GPU_mem * (1 - margin) - param_mem`, then computes KV cache capacity and maximum concurrent requests. Includes OOM detection with actionable suggestions. +- **Per-request KV cache tracking** (`vidur/entities/replica.py`) — Allocates and releases KV cache memory on a per-request basis, enabling accurate remaining-capacity queries at runtime. + +### References & Acknowledgments + +The GPU memory calculation module was developed with reference to the following works: + +- [InferSim](https://github.com/alibaba/InferSim) — Parameter counting and KV cache estimation methodology +- [DeepSeek V3 Parameter Size Analysis](https://yangwenbo.com/articles/deepseek-v3-parameter-size.html) — DeepSeek V3 MLA parameter derivation +- [DeepSeek V3 Parameter Derivation (Chinese)](https://zhuanlan.zhihu.com/p/21455638257) — Detailed MLA weight decomposition + +We gratefully acknowledge these resources for providing the foundational analysis that guided our implementation. --- ## Supported Models -+ **DeepSeek-V3-671B** (SimAI PP/EP communication、GPU memory allocation module adaptations in progress) -+ **Qwen3-Moe-235B**, **Qwen3-Next-80B** (SimAI PP/EP communication、GPU memory allocation module adaptations in progress) -+ **meta-llama/Meta-Llama-3-8B** / **Meta-Llama-3-70B** -+ **meta-llama/Llama-2-7b-hf** / **Llama-2-70b-hf** -+ **codellama/CodeLlama-34b-Instruct-hf** -+ **internlm/internlm-20b** -+ **Qwen/Qwen-72B** + +- **DeepSeek-V3-671B** (SimAI PP/EP communication and GPU memory allocation module adaptations in progress) +- **Qwen3-MoE-235B**, **Qwen3-Next-80B** (SimAI PP/EP communication and GPU memory allocation module adaptations in progress) +- **meta-llama/Meta-Llama-3-8B** / **Meta-Llama-3-70B** +- **meta-llama/Llama-2-7b-hf** / **Llama-2-70b-hf** +- **codellama/CodeLlama-34b-Instruct-hf** +- **internlm/internlm-20b** +- **Qwen/Qwen-72B** --- ## 📦 Environment Setup + ### 1. Create Conda Environment + ```bash conda env create -p ./env -f ./environment.yml ``` ### 2. (Optional) Update Dev Dependencies + ```bash conda env update -f environment-dev.yml ``` ### 3. Activate Environment + ```bash conda activate vidur ``` ### 4. Install Python Dependencies (Using Alibaba Cloud PyPI Mirror) + ```bash pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ pip install -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ @@ -66,13 +127,16 @@ pip install -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ --- -## ▶️ Running Example -### Run DeepSeek-671B **with** AICB -**Requirements: **SimAI and AICB Docker environment (see [README](../README.md) for setup instructions). +## ▶️ Running Examples + +### Run DeepSeek-671B with AICB + +**Requirements:** SimAI and AICB Docker environment (see [README](../README.md) for setup instructions). + +After setting up the environment, run the following commands: -After setting up the environment, run the following commands: +#### DeepSeek-671B with AICB (Fixed Length Generator) -#### Run DeepSeek-671B **with** AICB (Fixed Length Generator) ```bash cd SimAI/vidur-alibabacloud @@ -94,10 +158,11 @@ python -m vidur.main --replica_config_pd_p2p_comm_bandwidth 800 \ --replica_config_tensor_parallel_size 2 \ --replica_config_num_pipeline_stages 1 \ --replica_config_expert_model_parallel_size 8 \ - --random_forrest_execution_time_predictor_config_backend aicb + --random_forrest_execution_time_predictor_config_backend aicb ``` -#### Run DeepSeek-671B **with** AICB (Trace Length Generator) +#### DeepSeek-671B with AICB (Trace Length Generator) + ```bash cd SimAI/vidur-alibabacloud @@ -124,11 +189,9 @@ python -m vidur.main \ ``` > ✅ Full parameter descriptions are available via `python -m vidur.main -h`. -> - +### Run Llama-3-8B with SimAI Simulation -### Run Llama-3-8B **with** simai_simulation ```bash cd SimAI @@ -136,8 +199,8 @@ cd SimAI ./scripts/build.sh -c ns3 # Create network topo (Spectrum-X_128g_8gps_100Gbps_A100) -python3 ./astra-sim-alibabacloud/inputs/topo/gen_Topo_Template.py -topo Spectrum-X -g 128 -gt A100 -bw 100Gbps -nvbw 2400Gbps - +python3 ./astra-sim-alibabacloud/inputs/topo/gen_Topo_Template.py \ + -topo Spectrum-X -g 128 -gt A100 -bw 100Gbps -nvbw 2400Gbps cd SimAI/vidur-alibabacloud @@ -163,18 +226,16 @@ python -m vidur.main \ --random_forrest_execution_time_predictor_config_backend simai_simulation \ --random_forrest_execution_time_predictor_config_simai_dir ../ \ --random_forrest_execution_time_predictor_config_simai_simulation_topo ../Spectrum-X_128g_8gps_100Gbps_A100 \ - --random_forrest_execution_time_predictor_config_simai_simulation_config ../astra-sim-alibabacloud/inputs/config/SimAI.conf + --random_forrest_execution_time_predictor_config_simai_simulation_config ../astra-sim-alibabacloud/inputs/config/SimAI.conf ``` -> -> +### Run Llama-3-8B with SimAI Analytical -### Run Llama-3-8B **with** simai_analytical ```bash cd SimAI # Compile SimAI-Analytical -$ ./scripts/build.sh -c analytical +./scripts/build.sh -c analytical cd SimAI/vidur-alibabacloud @@ -200,10 +261,8 @@ python -m vidur.main \ --random_forrest_execution_time_predictor_config_backend simai_analytical ``` -> -> +### Run Llama-3-8B with Native Vidur [original] -### Run Llama-3-8B **with** native Vidur [original] ```bash cd SimAI/vidur-alibabacloud @@ -229,125 +288,200 @@ python -m vidur.main \ --random_forrest_execution_time_predictor_config_backend vidur ``` -> -> +### Run 4-Scenario Suite + +For a quick validation of all supported configurations, use the bundled test script: + +```bash +bash examples/vidur-ali-scenarios/run_scenarios.sh --all +``` + +See `bash examples/vidur-ali-scenarios/run_scenarios.sh --help` for details. +#### 4-Scenario Configuration +The following scenarios are pre-configured in `run_scenarios.sh`. All scenarios share the hardware configuration below. + +**Shared Hardware Configuration:** +- GPU: H20 (h20_dgx), NVLink: 1600 Gbps, RDMA: 800 Gbps +- PD P2P bandwidth: 800 Gbps, dtype: fp8 +- Request: Poisson QPS=100, 4 requests, fixed prefill=100 / decode=8 tokens + +| Scenario | Model | PD Separation | World Size | TP | PP | EP | Global Scheduler | +|----------|-------|---------------|------------|----|----|------------|------------------| +| 1 | Qwen3-Next-80B (MoE) | No | 32 (dp=32) | 1 | 1 | 1 (default) | lor | +| 2 | Qwen3-Next-80B (MoE) | Yes (P=2, D=6) | 8 | 1 | 1 | 1 (default) | split_wise | +| 3 | DeepSeek-671B (MoE) | Yes (P=2, D=6) | 8 | 8 | 1 | 8 | split_wise | +| 4 | Qwen3-MoE-235B (MoE) | Yes (P=2, D=6) | 8 | 4 | 1 | 4 | split_wise | + +> **Note:** All four models use Mixture-of-Experts (MoE) architecture. The EP column reflects the explicit `--replica_config_expert_model_parallel_size` value set in the script; scenarios without an explicit EP setting use the default value of 1. + +#### Output Files + +**Output path depends on how you run the simulation:** + +- **`run_scenarios.sh`** --- outputs to `examples/vidur-ali-scenarios/simulator_output/` +- **Direct `python -m vidur.main`** --- outputs to `./simulator_output/` (or the path specified by `--metrics_config_output_dir`) + +Each run produces the following directory: + +``` +// +├── request_metrics.csv # per-request metrics (see Key Output Interpretation) +├── chrome_trace.json # Chrome DevTools timeline trace (open at chrome://tracing) +├── config.json # snapshot of all simulation parameters +└── plots/ # per-metric CSV / JSON files (including but not limited to) + ├── request_e2e_time.csv + ├── prefill_e2e_time.csv + ├── pd_p2p_comm_time.csv + ├── replica_N_memory_usage.json + └── ... +``` + +> **Note:** The exact file list in `plots/` may vary across versions. +> Run-time logs (when using `run_scenarios.sh`) are saved separately to `examples/vidur-ali-scenarios/logs/scenario__.log`. --- ## 🔧 Key Input Parameter Reference + | Parameter | Default | Description | -| --- | --- | --- | -| `--replica_config_pd_p2p_comm_bandwidth` | 800 | Bandwidth (Gbps) for point-to-point communication between Prefill and Decode nodes in PD disaggregation | +|-----------|---------|-------------| +| `--replica_config_pd_p2p_comm_bandwidth` | 800 | Bandwidth (Gbps) for P2P communication between Prefill and Decode nodes in PD disaggregation | | `--replica_config_nvlink_bandwidth` | 1600 | NVLink bandwidth (Gbps) for TP/EP communications | | `--replica_config_rdma_bandwidth` | 800 | RDMA bandwidth (Gbps) for inter-node communication | | `--replica_config_pd_p2p_comm_dtype` | float16 | Data type for PD communication (`float16`, `float32`, etc.) | | `--poisson_request_interval_generator_config_qps` | 0.5 | Queries per second (QPS) for Poisson request generator | | `--synthetic_request_generator_config_num_requests` | 128 | Number of synthetic requests to generate | | `--length_generator_config_type` | fixed | Request length generator type (`fixed`, `trace`, etc.) | -| `--fixed_request_length_generator_config_prefill_tokens` | `2048` | Number of prefill tokens per request (only effective when `--length_generator_config_type=fixed`) | -| `--fixed_request_length_generator_config_decode_tokens` | `512` | Number of decode tokens per request (only effective when `--length_generator_config_type=fixed`) | +| `--fixed_request_length_generator_config_prefill_tokens` | 2048 | Number of prefill tokens per request (only effective when `--length_generator_config_type=fixed`) | +| `--fixed_request_length_generator_config_decode_tokens` | 512 | Number of decode tokens per request (only effective when `--length_generator_config_type=fixed`) | | `--trace_request_length_generator_config_max_tokens` | 4096 | Max tokens when using trace-based length generator | -| `--trace_request_length_generator_config_trace_file` | data/processed_traces/sharegpt_8k_filtered_stats_llama2_tokenizer.csv | Path to trace file for request lengths | +| `--trace_request_length_generator_config_trace_file` | `data/processed_traces/sharegpt_8k_filtered_stats_llama2_tokenizer.csv` | Path to trace file for request lengths | | `--interval_generator_config_type` | poisson | Inter-arrival time generator type | | `--cluster_config_num_replicas` | 1 | Total number of replicas (i.e., data parallelism degree) | -| `--replica_config_pd_node_ratio` | 0.5 | Ratio of P-nodes to （P-nodes + D-nodes） Fraction of replicas allocated as prefill (P) nodes. The remaining replicas are used as decode (D) nodes. For example, 0.5 means half of the replicas are prefill nodes and half are decode nodes (P:D = 1:1). | +| `--replica_config_pd_node_ratio` | 1 | Fraction of replicas allocated as prefill (P) nodes. 1 = MIXED mode (no PD separation). (0, 1) = PD separation enabled. E.g., 0.5 means P:D = 1:1. | | `--global_scheduler_config_type` | round_robin | Global scheduler type (`split_wise`, `round_robin`, etc.) | | `--replica_scheduler_config_type` | sarathi | Per-replica scheduler type | -| `--replica_config_model_name` | meta-llama/Llama-2-7b-hf | Model name (DeepSeek-671B, Qwen3-Moe-235B, Qwen3-Next-80B , etc.)
⚠️ **Note**: Vidur GPU Memory management module is still under adaptation for DeepSeek-671B, Qwen3-Moe-235B, Qwen3-Next-80B | +| `--replica_config_model_name` | meta-llama/Llama-2-7b-hf | Model name (DeepSeek-671B, Qwen3-MoE-235B, Qwen3-Next-80B, etc.) | | `--replica_config_tensor_parallel_size` | 1 | Tensor parallelism size (TP) | | `--replica_config_num_pipeline_stages` | 1 | Number of pipeline stages (PP) | | `--replica_config_expert_model_parallel_size` | 1 | Expert model parallelism size (EP) | -| `--random_forrest_execution_time_predictor_config_backend` | vidur | Backend for execution time prediction
('vidur', 'simai_simulation', 'simai_analytical','aicb', etc.)
⚠️ **Note**: `simai_simulation` and `simai_analytical` currently only model TP communication and do not support pipeline or expert parallelism | -| `--random_forrest_execution_time_predictor_config_simai_dir` | `'../'` | Root directory of the SimAI simulator（default: `../`)
（only effective when `--random_forrest_execution_time_predictor_config_backend simai_simulation`） | -| `--random_forrest_execution_time_predictor_config_simai_simulation_topo` | `'../example/topo'` | Path to SimAI topology file (e.g., `'../Spectrum-X_128g_8gps_100Gbps_A100'`)（only effective when `--random_forrest_execution_time_predictor_config_backend simai_simulation`） | -| `--random_forrest_execution_time_predictor_config_simai_simulation_config` | `'../astra-sim-alibabacloud/inputs/config/SimAI.conf'` | Path to SimAI configuration file (e.g., `'../astra-sim-alibabacloud/inputs/config/SimAI.conf'`)
（only effective when `--random_forrest_execution_time_predictor_config_backend simai_simulation`） | +| `--random_forrest_execution_time_predictor_config_backend` | vidur | Backend for execution time prediction (`vidur`, `simai_simulation`, `simai_analytical`, `aicb`, etc.). **Note:** `simai_simulation` and `simai_analytical` currently only model TP communication and do not support pipeline or expert parallelism. | +| `--random_forrest_execution_time_predictor_config_simai_dir` | `../` | Root directory of the SimAI simulator (only effective when backend = `simai_simulation`) | +| `--random_forrest_execution_time_predictor_config_simai_simulation_topo` | `../example/topo` | Path to SimAI topology file (only effective when backend = `simai_simulation`) | +| `--random_forrest_execution_time_predictor_config_simai_simulation_config` | `../astra-sim-alibabacloud/inputs/config/SimAI.conf` | Path to SimAI configuration file (only effective when backend = `simai_simulation`) | + +### PD Separation Parameters +When `pd_node_ratio` < 1, the following optional parameters become effective: + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `--replica_config_prefill_tensor_parallel_size` | None | Prefill-specific TP size. Falls back to `tensor_parallel_size` if not set. | +| `--replica_config_prefill_num_pipeline_stages` | None | Prefill-specific PP size. Falls back to `num_pipeline_stages` if not set. | +| `--replica_config_decode_tensor_parallel_size` | None | Decode-specific TP size. Falls back to `tensor_parallel_size` if not set. | +| `--replica_config_decode_num_pipeline_stages` | None | Decode-specific PP size. Falls back to `num_pipeline_stages` if not set. | +| `--replica_config_num_prefill_replicas` | None | Directly specify prefill replica count (takes priority over `pd_node_ratio`). | + +**Example: DeepSeek-671B with PD separation (P:D = 2:6)** + +```bash +python -m vidur.main \ + --replica_config_model_name deepseek-671B \ + --replica_config_device h20 \ + --replica_config_network_device h20_dgx \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --replica_config_tensor_parallel_size 8 \ + --replica_config_num_pipeline_stages 1 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --random_forrest_execution_time_predictor_config_backend aicb +``` --- ## 📊 Key Output Interpretation + Simulation results are saved to: -```plain +``` ./simulator_output/YYYY-MM-DD_HH-MM-SS-XXXXXX/request_metrics.csv ``` ### Key Columns in `request_metrics.csv` + | Column | Meaning | -| --- | --- | +|--------|---------| | `arrived_at` / `prefill_arrived_at` | Timestamp when the request entered the system (in seconds). | -| `scheduled_at` | Timestamp when the request was first scheduled by the scheduler and began execution (in seconds). | +| `scheduled_at` | Timestamp when the request was first scheduled and began execution (in seconds). | | `prefill_completed_at` | Timestamp when the Prefill phase completed and the first output token was generated. | -| `decode_arrived_at` | Timestamp when the Decode phase started. In non-PD-Disaggregated setup, this typically equals `prefill_completed_at`. In PD-Disaggregated setup, it is `prefill_completed_at + pd_p2p_comm_time`. | -| `decode_time` | Duration of the Decode phase (in seconds), computed as `completed_at - decode_arrived_at` (equivalently: `request_e2e_time - prefill_e2e_time`). | -| `prefill_replica_id` | Replica ID that executed the Prefill phase (in PD-Disaggregated setup). | -| `decode_replica_id` | Replica ID that executed the Decode phase (in PD-Disaggregated setup). | +| `decode_arrived_at` | Timestamp when the Decode phase started. In non-PD-disaggregated setup, this typically equals `prefill_completed_at`. In PD-disaggregated setup, it is `prefill_completed_at + pd_p2p_comm_time`. | +| `decode_time` | Duration of the Decode phase (in seconds), computed as `completed_at - decode_arrived_at`. | +| `prefill_replica_id` | Replica ID that executed the Prefill phase (in PD-disaggregated setup). | +| `decode_replica_id` | Replica ID that executed the Decode phase (in PD-disaggregated setup). | | `request_num_prefill_tokens` | Number of input tokens (i.e., prompt length). | | `request_num_decode_tokens` | Number of output tokens (i.e., generation length). | -| `pd_p2p_comm_size` | Point-to-point communication size (in bytes) of data transferred from the Prefill node to the Decode node (KV cache, etc.) in PD-Disaggregated setup. | -| `pd_p2p_comm_time` | Point-to-point communication time (in seconds) between Prefill and Decode nodes in PD-Disaggregated setup. | +| `pd_p2p_comm_size` | P2P communication size (in bytes) of data transferred from the Prefill node to the Decode node (KV cache, etc.) in PD-disaggregated setup. | +| `pd_p2p_comm_time` | P2P communication time (in seconds) between Prefill and Decode nodes in PD-disaggregated setup. | | `completed_at` | Timestamp when the request finished processing. | | `request_execution_time` | Total actual execution time (in seconds), excluding delays due to preemption or pipeline bubbles. | | `request_preemption_time` | Time (in seconds) spent waiting due to scheduler preemption, pipeline bubbles, or other non-execution gaps. | | `request_scheduling_delay` | Scheduling delay before execution: `scheduled_at - arrived_at` (in seconds). | | `request_e2e_time` | End-to-end latency: `completed_at - arrived_at` (in seconds). | | `prefill_e2e_time` | Time To First Token (TTFT): `prefill_completed_at - arrived_at` (in seconds). | -| `tbt` | Time Between Tokens (TBT), also known as Time Per Output Token (TPOT). Computed as: `decode_time / request_num_decode_tokens` or equivalently: `(request_e2e_time - prefill_e2e_time) / request_num_decode_tokens` (in seconds/token). | +| `tbt` | Time Between Tokens (TBT), also known as TPOT. Computed as: `decode_time / request_num_decode_tokens` (in seconds/token). | +**Notes:** - **Notes**: +- All time-related fields are in **seconds (s)**, based on monotonic clock or Unix timestamps. +- In non-PD-separated deployments, `prefill_replica_id` and `decode_replica_id` are typically identical. +- If `request_num_decode_tokens = 0`, `tbt` is undefined (may be recorded as `NaN` or `0`). +- TBT is not yet logged in `request_metrics.csv`; it can be computed manually for now. -+ All time-related fields are in **seconds (s)**, based on monotonic clock or Unix timestamps. -+ In non-PD-separated deployments, `prefill_replica_id` and `decode_replica_id` are typically identical. -+ If `request_num_decode_tokens = 0`, `tbt` is undefined (may be recorded as `NaN` or `0`). -+ **TBT is not yet logged in request_metrics.csv; it can be computed manually for now.** +### Sample Row (`request_metrics.csv`) -### Sample Row (request_metrics.csv) -```plain +``` Request Id,request_e2e_time,...,arrived_at,prefill_arrived_at,scheduled_at,prefill_completed_at,decode_arrived_at,completed_at,...,prefill_replica_id,decode_replica_id,pd_p2p_comm_size,pd_p2p_comm_time,... 0,0.03607,...,0.0102006,0.0102006,0.0102006,0.0102265,0.0433997,0.0462744,...,0,2,3561947136,0.0331732,... ``` --- -## ⚠️ Known Issue: Plotting Warning +## ⚠️ Known Issues + +### Plotting Warning + You may see this error at exit: -```plain +``` RuntimeError: Kaleido requires Google Chrome to be installed. ``` -This occurs because the simulator tries to generate PNG plots but lacks Chrome. -✅ **Important**: This **does NOT affect** the generation of `request_metrics.csv`. - -### Solutions: -1. **Ignore it** – CSV output is unaffected. -2. **Install Chrome**: +This occurs because the simulator tries to generate PNG plots but lacks Chrome. +**Important:** This does **NOT** affect the generation of `request_metrics.csv`. -```bash -plotly_get_chrome -``` +**Solutions:** +1. **Ignore it** — CSV output is unaffected. +2. **Install Chrome:** + ```bash + plotly_get_chrome + ``` 3. **Disable plotting** (not recommended): Comment out these lines in `vidur/simulator.py`: - -```python -# self._metric_store.plot() -# logger.info("Metrics written") -``` - -> ⚠️ Disabling plotting will skip all visual outputs and request_metrics.csv. -> + ```python + # self._metric_store.plot() + # logger.info("Metrics written") + ``` + > Disabling plotting will skip all visual outputs and `request_metrics.csv`. --- ## 📚 Help + View all CLI options: ```bash python -m vidur.main -h ``` - ---- - diff --git a/vidur-alibabacloud/README_CN.md b/vidur-alibabacloud/README_CN.md new file mode 100644 index 00000000..372f8303 --- /dev/null +++ b/vidur-alibabacloud/README_CN.md @@ -0,0 +1,458 @@ +

+ 中文  ｜  English +

+ +# Vidur-AlibabaCloud + +[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/) +[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) + +Vidur（[原版](https://github.com/microsoft/vidur)）是一个大语言模型（LLM）推理系统的模拟框架。 +**Vidur-AlibabaCloud**（本仓库）是针对阿里云 **SimAI** 场景优化的定制版本。支持 **Prefill–Decode（PD）分离**等高级特性，并针对 **DeepSeek-V3-671B**、**Qwen3-MoE-235B**、**Qwen3-Next-80B** 等 SOTA 大模型进行了专门适配。 + +--- + +## 目录 + +- [主要特性](#主要特性) +- [GPU 显存计算模块](#gpu-显存计算模块) +- [支持的模型](#支持的模型) +- [📦 环境配置](#-环境配置) +- [▶️ 运行示例](#️-运行示例) + - [四场景配置说明](#四场景配置说明) + - [输出文件说明](#输出文件说明) +- [🔧 关键输入参数参考](#-关键输入参数参考) +- [📊 输出结果解读](#-输出结果解读) +- [⚠️ 已知问题](#️-已知问题) +- [📚 帮助](#-帮助) + +--- + +## 主要特性 + +- **Prefill–Decode（PD）分离** — 支持 prefill 和 decode 阶段在不同节点运行，实现弹性资源分配和性能隔离。 + （参考 [splitwise-sim](https://github.com/Mutinifni/splitwise-sim)） +- **灵活的并行策略** — 支持： + - **数据并行（DP）** + - **张量并行（TP）** + - **流水线并行（PP）** + - **专家并行（EP）**（适配中） + + 同时支持 **Dense** 模型和 **混合专家（MoE）** 模型（MoE 适配中）。 +- **多种执行时间预测后端** — 可选： + - **AICB/AIOB** — 部分支持 DeepSeek-V3-671B、Qwen3-MoE-235B、Qwen3-Next-80B 的计算核与 TP、DP、PP、EP 通信量建模 + - **SimAI 仿真（Simulation）** — 基于 SimAI NS-3 的网络通信全栈仿真（支持 TP） + - **SimAI 解析（Analytical）** — SimAI 解析性能模型（支持 TP） + - **原版 Vidur [original]** — 支持 TP、DP、PP +- **负载生成与回放** — 支持真实 trace 回放，或使用固定/泊松分布生成合成请求。 +- **细粒度指标** — 记录： + - TTFT — 首 token 时延 + - TBT / TPOT — 相邻 token 时延 / 每输出 token 耗时 + - 端到端延迟 + - 通信开销 + - 计算开销 + - 调度延迟 + +--- + +## GPU 显存计算模块 + +本模块为现代 MoE（混合专家）模型的推理仿真提供精确的 GPU 显存估算，涵盖**模型参数显存**、**KV Cache 显存**以及 Prefill–Decode（PD）分离架构下的**最大批处理量**计算。 + +### 支持的注意力架构 + +| 架构 | 模型 | 说明 | +|---|---|---| +| **MLA**（多头潜在注意力） | DeepSeek-V3-671B | 使用 LoRA 压缩的 KV Cache（`kv_lora_rank` + `qk_rope_head_dim`），显著降低显存占用 | +| **MHA / GQA**（多头 / 分组查询注意力） | Qwen3-MoE-235B | 标准 KV Cache，每 token 每层使用 `num_kv_heads * head_dim` | +| **混合全注意力 + 线性注意力** | Qwen3-Next-80B | 每 4 层交替使用全注意力和线性（GDN）注意力 | + +### 核心组件 + +- **`ParamCounter`**（`vidur/utils/param_counter.py`）— 计算每层和每设备的参数量，支持 MLA、MHA/GQA、线性注意力和 MoE 专家权重，支持 FP8 量化。在 PD 分离架构下，根据 `prefill_world_size` / `decode_world_size` 分别返回 `(total_params, prefill_params, decode_params)` 三元组。 +- **`MemoryPlanner`**（`vidur/scheduler/utils/memory_planner.py`）— 规划 GPU 显存预算：`available = GPU_mem * (1 - margin) - param_mem`，计算 KV Cache 容量和最大并发请求数，包含 OOM 检测与建议输出。 +- **逐请求 KV Cache 追踪**（`vidur/entities/replica.py`）— 按请求粒度分配和释放 KV Cache 显存，支持运行时精确查询剩余容量。 + +### 参考与致谢 + +本 GPU 显存计算模块的开发参考了以下工作： + +- [InferSim](https://github.com/alibaba/InferSim) — 参数量计算与 KV Cache 估算方法论 +- [DeepSeek V3 Parameter Size Analysis](https://yangwenbo.com/articles/deepseek-v3-parameter-size.html) — DeepSeek V3 MLA 参数推导 +- [DeepSeek V3 参数推导详解](https://zhuanlan.zhihu.com/p/21455638257) — MLA 权重分解详细分析 + +衷心感谢以上资源为我们的实现提供了基础性的分析与指导。 + +--- + +## 支持的模型 + +- **DeepSeek-V3-671B**（SimAI PP/EP 通信及 GPU 显存管理模块适配中） +- **Qwen3-MoE-235B**、**Qwen3-Next-80B**（SimAI PP/EP 通信及 GPU 显存管理模块适配中） +- **meta-llama/Meta-Llama-3-8B** / **Meta-Llama-3-70B** +- **meta-llama/Llama-2-7b-hf** / **Llama-2-70b-hf** +- **codellama/CodeLlama-34b-Instruct-hf** +- **internlm/internlm-20b** +- **Qwen/Qwen-72B** + +--- + +## 📦 环境配置 + +### 1. 创建 Conda 环境 + +```bash +conda env create -p ./env -f ./environment.yml +``` + +### 2.（可选）更新开发依赖 + +```bash +conda env update -f environment-dev.yml +``` + +### 3. 激活环境 + +```bash +conda activate vidur +``` + +### 4. 安装 Python 依赖（使用阿里云 PyPI 镜像） + +```bash +pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ +pip install -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ +``` + +--- + +## ▶️ 运行示例 + +### 使用 AICB 运行 DeepSeek-671B + +**前置条件：** 需要 SimAI 和 AICB Docker 环境（参见 [README](../README.md) 了解搭建方法）。 + +完成环境配置后，运行以下命令： + +#### DeepSeek-671B + AICB（固定长度生成器） + +```bash +cd SimAI/vidur-alibabacloud + +python -m vidur.main --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype float32 \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 5 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 1024 \ + --fixed_request_length_generator_config_decode_tokens 10 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --cluster_config_num_replicas 4 \ + --replica_config_pd_node_ratio 0.5 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name deepseek-671B \ + --replica_config_tensor_parallel_size 2 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 8 \ + --random_forrest_execution_time_predictor_config_backend aicb +``` + +#### DeepSeek-671B + AICB（Trace 长度生成器） + +```bash +cd SimAI/vidur-alibabacloud + +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype float32 \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 10 \ + --length_generator_config_type trace \ + --trace_request_length_generator_config_max_tokens 1024 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --interval_generator_config_type poisson \ + --cluster_config_num_replicas 4 \ + --replica_config_pd_node_ratio 0.5 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name deepseek-671B \ + --replica_config_tensor_parallel_size 2 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 8 \ + --random_forrest_execution_time_predictor_config_backend aicb +``` + +> ✅ 完整参数说明可通过 `python -m vidur.main -h` 查看。 + +### 使用 SimAI 仿真运行 Llama-3-8B + +```bash +cd SimAI + +# 编译 SimAI-Simulation（ns3） +./scripts/build.sh -c ns3 + +# 生成网络拓扑（Spectrum-X_128g_8gps_100Gbps_A100） +python3 ./astra-sim-alibabacloud/inputs/topo/gen_Topo_Template.py \ + -topo Spectrum-X -g 128 -gt A100 -bw 100Gbps -nvbw 2400Gbps + +cd SimAI/vidur-alibabacloud + +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype float32 \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 10 \ + --length_generator_config_type trace \ + --trace_request_length_generator_config_max_tokens 2048 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --interval_generator_config_type poisson \ + --cluster_config_num_replicas 4 \ + --replica_config_pd_node_ratio 0.5 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name meta-llama/Meta-Llama-3-8B \ + --replica_config_tensor_parallel_size 4 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 1 \ + --random_forrest_execution_time_predictor_config_backend simai_simulation \ + --random_forrest_execution_time_predictor_config_simai_dir ../ \ + --random_forrest_execution_time_predictor_config_simai_simulation_topo ../Spectrum-X_128g_8gps_100Gbps_A100 \ + --random_forrest_execution_time_predictor_config_simai_simulation_config ../astra-sim-alibabacloud/inputs/config/SimAI.conf +``` + +### 使用 SimAI 解析模型运行 Llama-3-8B + +```bash +cd SimAI + +# 编译 SimAI-Analytical +./scripts/build.sh -c analytical + +cd SimAI/vidur-alibabacloud + +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype float32 \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 10 \ + --length_generator_config_type trace \ + --trace_request_length_generator_config_max_tokens 2048 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --interval_generator_config_type poisson \ + --cluster_config_num_replicas 4 \ + --replica_config_pd_node_ratio 0.5 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name meta-llama/Meta-Llama-3-8B \ + --replica_config_tensor_parallel_size 4 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 1 \ + --random_forrest_execution_time_predictor_config_backend simai_analytical +``` + +### 使用原版 Vidur 运行 Llama-3-8B + +```bash +cd SimAI/vidur-alibabacloud + +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype float32 \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 10 \ + --length_generator_config_type trace \ + --trace_request_length_generator_config_max_tokens 2048 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --interval_generator_config_type poisson \ + --cluster_config_num_replicas 4 \ + --replica_config_pd_node_ratio 0.5 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name meta-llama/Meta-Llama-3-8B \ + --replica_config_tensor_parallel_size 4 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 1 \ + --random_forrest_execution_time_predictor_config_backend vidur +``` + +### 运行四场景套件 + +使用内置脚本快速验证所有支持的配置： + +```bash +bash examples/vidur-ali-scenarios/run_scenarios.sh --all +``` + +详细信息请运行 `bash examples/vidur-ali-scenarios/run_scenarios.sh --help`。 + +#### 四场景配置说明 + +以下场景已在 `run_scenarios.sh` 中预配置，所有场景共享下方硬件配置。 + +**共用硬件配置：** +- GPU：H20（h20_dgx），NVLink：1600 Gbps，RDMA：800 Gbps +- PD P2P 带宽：800 Gbps，数据类型：fp8 +- 请求生成：Poisson QPS=100，4 requests，固定 prefill=100 / decode=8 tokens + +| 场景 | 模型 | PD 分离 | World Size | TP | PP | EP | 全局调度器 | +|------|------|---------|------------|----|----|------------|------------| +| 1 | Qwen3-Next-80B (MoE) | 无 | 32 (dp=32) | 1 | 1 | 1（默认） | lor | +| 2 | Qwen3-Next-80B (MoE) | 是（P=2, D=6） | 8 | 1 | 1 | 1（默认） | split_wise | +| 3 | DeepSeek-671B (MoE) | 是（P=2, D=6） | 8 | 8 | 1 | 8 | split_wise | +| 4 | Qwen3-MoE-235B (MoE) | 是（P=2, D=6） | 8 | 4 | 1 | 4 | split_wise | + +> **说明：** 四个模型均使用混合专家（MoE）架构。EP 列反映脚本中 `--replica_config_expert_model_parallel_size` 的显式设定值；未显式指定时使用默认值 1。 + +#### 输出文件说明 + +**输出路径取决于运行方式：** + +- **`run_scenarios.sh`** --- 输出到 `examples/vidur-ali-scenarios/simulator_output/` +- **直接 `python -m vidur.main`** --- 输出到 `./simulator_output/`（或通过 `--metrics_config_output_dir` 指定的路径） + +每次运行产生如下目录： + +``` +// +├── request_metrics.csv # 逐请求指标（参见"输出结果解读"） +├── chrome_trace.json # Chrome DevTools 时间轴 trace（可在 chrome://tracing 打开） +├── config.json # 本次仿真的全部参数快照 +└── plots/ # 逐指标 CSV / JSON 文件（包括但不限于） + ├── request_e2e_time.csv + ├── prefill_e2e_time.csv + ├── pd_p2p_comm_time.csv + ├── replica_N_memory_usage.json + └── ... +``` + +> **说明：** `plots/` 中的具体文件列表可能因版本不同而变化。 +> 使用 `run_scenarios.sh` 时，运行日志另存于 `examples/vidur-ali-scenarios/logs/scenario__.log`。 + +--- + +## 🔧 关键输入参数参考 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `--replica_config_pd_p2p_comm_bandwidth` | 800 | PD 分离中 Prefill 节点与 Decode 节点间 P2P 通信带宽（Gbps） | +| `--replica_config_nvlink_bandwidth` | 1600 | TP/EP 通信使用的 NVLink 带宽（Gbps） | +| `--replica_config_rdma_bandwidth` | 800 | 节点间通信使用的 RDMA 带宽（Gbps） | +| `--replica_config_pd_p2p_comm_dtype` | float16 | PD 通信数据类型（`float16`、`float32` 等） | +| `--poisson_request_interval_generator_config_qps` | 0.5 | 泊松请求生成器的 QPS（每秒请求数） | +| `--synthetic_request_generator_config_num_requests` | 128 | 合成请求总数 | +| `--length_generator_config_type` | fixed | 请求长度生成器类型（`fixed`、`trace` 等） | +| `--fixed_request_length_generator_config_prefill_tokens` | 2048 | 每请求的 prefill token 数（仅在 `--length_generator_config_type=fixed` 时生效） | +| `--fixed_request_length_generator_config_decode_tokens` | 512 | 每请求的 decode token 数（仅在 `--length_generator_config_type=fixed` 时生效） | +| `--trace_request_length_generator_config_max_tokens` | 4096 | 使用 trace 长度生成器时的最大 token 数 | +| `--trace_request_length_generator_config_trace_file` | `data/processed_traces/sharegpt_8k_filtered_stats_llama2_tokenizer.csv` | trace 文件路径 | +| `--interval_generator_config_type` | poisson | 请求到达间隔生成器类型 | +| `--cluster_config_num_replicas` | 1 | replica 总数（即数据并行度） | +| `--replica_config_pd_node_ratio` | 0.5 | 分配为 Prefill（P）节点的 replica 比例，其余为 Decode（D）节点。例如 0.5 表示 P:D = 1:1。 | +| `--global_scheduler_config_type` | round_robin | 全局调度器类型（`split_wise`、`round_robin` 等） | +| `--replica_scheduler_config_type` | sarathi | 单 replica 调度器类型 | +| `--replica_config_model_name` | meta-llama/Llama-2-7b-hf | 模型名称（DeepSeek-671B、Qwen3-MoE-235B、Qwen3-Next-80B 等） | +| `--replica_config_tensor_parallel_size` | 1 | 张量并行大小（TP） | +| `--replica_config_num_pipeline_stages` | 1 | 流水线阶段数（PP） | +| `--replica_config_expert_model_parallel_size` | 1 | 专家并行大小（EP） | +| `--random_forrest_execution_time_predictor_config_backend` | vidur | 执行时间预测后端（`vidur`、`simai_simulation`、`simai_analytical`、`aicb` 等）。**注意：** `simai_simulation` 和 `simai_analytical` 当前仅建模 TP 通信，不支持流水线或专家并行。 | +| `--random_forrest_execution_time_predictor_config_simai_dir` | `../` | SimAI 模拟器根目录（仅在 backend = `simai_simulation` 时生效） | +| `--random_forrest_execution_time_predictor_config_simai_simulation_topo` | `../example/topo` | SimAI 拓扑文件路径（仅在 backend = `simai_simulation` 时生效） | +| `--random_forrest_execution_time_predictor_config_simai_simulation_config` | `../astra-sim-alibabacloud/inputs/config/SimAI.conf` | SimAI 配置文件路径（仅在 backend = `simai_simulation` 时生效） | + +--- + +## 📊 输出结果解读 + +仿真结果保存于： + +``` +./simulator_output/YYYY-MM-DD_HH-MM-SS-XXXXXX/request_metrics.csv +``` + +### `request_metrics.csv` 关键列说明 + +| 列名 | 含义 | +|------|------| +| `arrived_at` / `prefill_arrived_at` | 请求进入系统的时间戳（秒）。 | +| `scheduled_at` | 请求首次被调度并开始执行的时间戳（秒）。 | +| `prefill_completed_at` | Prefill 阶段完成、生成第一个输出 token 的时间戳。 | +| `decode_arrived_at` | Decode 阶段开始的时间戳。非 PD 分离场景下通常等于 `prefill_completed_at`；PD 分离场景下为 `prefill_completed_at + pd_p2p_comm_time`。 | +| `decode_time` | Decode 阶段持续时间（秒），计算公式：`completed_at - decode_arrived_at`。 | +| `prefill_replica_id` | 执行 Prefill 阶段的 replica ID（PD 分离场景下）。 | +| `decode_replica_id` | 执行 Decode 阶段的 replica ID（PD 分离场景下）。 | +| `request_num_prefill_tokens` | 输入 token 数（即 prompt 长度）。 | +| `request_num_decode_tokens` | 输出 token 数（即生成长度）。 | +| `pd_p2p_comm_size` | PD 分离场景下，从 Prefill 节点传输至 Decode 节点的数据量（字节，含 KV Cache 等）。 | +| `pd_p2p_comm_time` | PD 分离场景下，Prefill 节点与 Decode 节点间 P2P 通信耗时（秒）。 | +| `completed_at` | 请求处理完成的时间戳。 | +| `request_execution_time` | 实际执行总时间（秒），不含抢占或流水线气泡导致的等待。 | +| `request_preemption_time` | 因调度器抢占、流水线气泡或其他非执行间隔导致的等待时间（秒）。 | +| `request_scheduling_delay` | 执行前的调度延迟：`scheduled_at - arrived_at`（秒）。 | +| `request_e2e_time` | 端到端延迟：`completed_at - arrived_at`（秒）。 | +| `prefill_e2e_time` | 首 token 时延（TTFT）：`prefill_completed_at - arrived_at`（秒）。 | +| `tbt` | 相邻 token 时延（TBT / TPOT）。计算公式：`decode_time / request_num_decode_tokens`（秒/token）。 | + +**说明：** + +- 所有时间字段单位均为**秒（s）**，基于单调时钟或 Unix 时间戳。 +- 非 PD 分离部署中，`prefill_replica_id` 与 `decode_replica_id` 通常相同。 +- 若 `request_num_decode_tokens = 0`，则 `tbt` 未定义（可能记录为 `NaN` 或 `0`）。 +- `tbt` 暂未写入 `request_metrics.csv`，目前需手动计算。 + +### 示例行（`request_metrics.csv`） + +``` +Request Id,request_e2e_time,...,arrived_at,prefill_arrived_at,scheduled_at,prefill_completed_at,decode_arrived_at,completed_at,...,prefill_replica_id,decode_replica_id,pd_p2p_comm_size,pd_p2p_comm_time,... +0,0.03607,...,0.0102006,0.0102006,0.0102006,0.0102265,0.0433997,0.0462744,...,0,2,3561947136,0.0331732,... +``` + +--- + +## ⚠️ 已知问题 + +### 绘图警告 + +退出时可能出现以下错误： + +``` +RuntimeError: Kaleido requires Google Chrome to be installed. +``` + +这是因为模拟器尝试生成 PNG 图表但缺少 Chrome。 +**重要：** 此问题**不影响** `request_metrics.csv` 的生成。 + +**解决方案：** + +1. **忽略** — CSV 输出不受影响。 +2. **安装 Chrome：** + ```bash + plotly_get_chrome + ``` +3. **禁用绘图**（不推荐）：注释掉 `vidur/simulator.py` 中的以下行： + ```python + # self._metric_store.plot() + # logger.info("Metrics written") + ``` + > 禁用绘图将跳过所有可视化输出及 `request_metrics.csv`。 + +--- + +## 📚 帮助 + +查看所有 CLI 选项： + +```bash +python -m vidur.main -h +``` diff --git a/vidur-alibabacloud/data/hf_configs/deepseek_R1_0528_config.json b/vidur-alibabacloud/data/hf_configs/deepseek_R1_0528_config.json new file mode 100644 index 00000000..c482e777 --- /dev/null +++ b/vidur-alibabacloud/data/hf_configs/deepseek_R1_0528_config.json @@ -0,0 +1,67 @@ +{ + "architectures": [ + "DeepseekV3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoConfig": "configuration_deepseek.DeepseekV3Config", + "AutoModel": "modeling_deepseek.DeepseekV3Model", + "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" + }, + "bos_token_id": 0, + "eos_token_id": 1, + "ep_size": 1, + "first_k_dense_replace": 3, + "hidden_act": "silu", + "hidden_size": 7168, + "initializer_range": 0.02, + "intermediate_size": 18432, + "kv_lora_rank": 512, + "max_position_embeddings": 163840, + "model_type": "deepseek_v3", + "moe_intermediate_size": 2048, + "moe_layer_freq": 1, + "n_group": 8, + "n_routed_experts": 256, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 61, + "num_key_value_heads": 128, + "num_nextn_predict_layers": 1, + "q_lora_rank": 1536, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "weight_block_size": [ + 128, + 128 + ] + }, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "beta_fast": 32, + "beta_slow": 1, + "factor": 40, + "mscale": 1.0, + "mscale_all_dim": 1.0, + "original_max_position_embeddings": 4096, + "type": "yarn" + }, + "rope_theta": 10000, + "routed_scaling_factor": 2.5, + "scoring_func": "sigmoid", + "tie_word_embeddings": false, + "topk_group": 4, + "topk_method": "noaux_tc", + "torch_dtype": "bfloat16", + "transformers_version": "4.46.3", + "use_cache": true, + "v_head_dim": 128, + "vocab_size": 129280 +} diff --git a/vidur-alibabacloud/data/hf_configs/deepseek_v3_config.json b/vidur-alibabacloud/data/hf_configs/deepseek_v3_config.json new file mode 100644 index 00000000..0ef21d65 --- /dev/null +++ b/vidur-alibabacloud/data/hf_configs/deepseek_v3_config.json @@ -0,0 +1,70 @@ +{ + "architectures": [ + "DeepseekV3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoConfig": "configuration_deepseek.DeepseekV3Config", + "AutoModel": "modeling_deepseek.DeepseekV3Model", + "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" + }, + "aux_loss_alpha": 0.001, + "bos_token_id": 0, + "eos_token_id": 1, + "ep_size": 1, + "first_k_dense_replace": 3, + "hidden_act": "silu", + "hidden_size": 7168, + "initializer_range": 0.02, + "intermediate_size": 18432, + "kv_lora_rank": 512, + "max_position_embeddings": 163840, + "model_type": "deepseek_v3", + "moe_intermediate_size": 2048, + "moe_layer_freq": 1, + "n_group": 8, + "num_routed_experts": 256, + "num_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 61, + "num_key_value_heads": 128, + "num_nextn_predict_layers": 1, + "pretraining_tp": 1, + "q_lora_rank": 1536, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "weight_block_size": [ + 128, + 128 + ] + }, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "beta_fast": 32, + "beta_slow": 1, + "factor": 40, + "mscale": 1.0, + "mscale_all_dim": 1.0, + "original_max_position_embeddings": 4096, + "type": "yarn" + }, + "rope_theta": 10000, + "routed_scaling_factor": 2.5, + "scoring_func": "sigmoid", + "seq_aux": true, + "tie_word_embeddings": false, + "topk_group": 4, + "topk_method": "noaux_tc", + "torch_dtype": "bfloat16", + "transformers_version": "4.46.3", + "use_cache": true, + "v_head_dim": 128, + "vocab_size": 129280 +} diff --git a/vidur-alibabacloud/data/hf_configs/qwen3-235B-A22B_FP8_config.json b/vidur-alibabacloud/data/hf_configs/qwen3-235B-A22B_FP8_config.json new file mode 100644 index 00000000..01e9c6f7 --- /dev/null +++ b/vidur-alibabacloud/data/hf_configs/qwen3-235B-A22B_FP8_config.json @@ -0,0 +1,49 @@ +{ + "architectures": [ + "Qwen3MoeForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "decoder_sparse_step": 1, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 16384, + "max_position_embeddings": 65536, + "max_window_layers": 94, + "mlp_only_layers": [], + "model_type": "qwen2_moe", + "moe_intermediate_size": 1536, + "norm_topk_prob": true, + "num_attention_heads": 64, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 94, + "num_key_value_heads": 4, + "output_router_logits": false, + "qkv_bias": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000.0, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 0, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.49.0.dev0", + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 151936, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "weight_block_size": [ + 128, + 128 + ] + } +} diff --git a/vidur-alibabacloud/data/hf_configs/qwen3-235B-A22B_config.json b/vidur-alibabacloud/data/hf_configs/qwen3-235B-A22B_config.json new file mode 100644 index 00000000..bb4e2521 --- /dev/null +++ b/vidur-alibabacloud/data/hf_configs/qwen3-235B-A22B_config.json @@ -0,0 +1,38 @@ +{ + "architectures": [ + "Qwen3MoeForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "decoder_sparse_step": 1, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "max_position_embeddings": 262144, + "max_window_layers": 94, + "mlp_only_layers": [], + "model_type": "qwen3_moe", + "moe_intermediate_size": 1536, + "norm_topk_prob": true, + "num_attention_heads": 64, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 94, + "num_key_value_heads": 4, + "output_router_logits": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 5000000, + "router_aux_loss_coef": 0.001, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/vidur-alibabacloud/data/hf_configs/qwen3-30B-A3B_config.json b/vidur-alibabacloud/data/hf_configs/qwen3-30B-A3B_config.json new file mode 100644 index 00000000..29951baa --- /dev/null +++ b/vidur-alibabacloud/data/hf_configs/qwen3-30B-A3B_config.json @@ -0,0 +1,38 @@ +{ + "architectures": [ + "Qwen3MoeForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "decoder_sparse_step": 1, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "max_position_embeddings": 40960, + "max_window_layers": 48, + "mlp_only_layers": [], + "model_type": "qwen3_moe", + "moe_intermediate_size": 768, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 4, + "output_router_logits": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000.0, + "router_aux_loss_coef": 0.001, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/vidur-alibabacloud/data/hf_configs/qwen3-8B_config.json b/vidur-alibabacloud/data/hf_configs/qwen3-8B_config.json new file mode 100644 index 00000000..79c4558e --- /dev/null +++ b/vidur-alibabacloud/data/hf_configs/qwen3-8B_config.json @@ -0,0 +1,30 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/vidur-alibabacloud/data/hf_configs/qwen3-next-80B-A3B_Instruct_FP8_config.json b/vidur-alibabacloud/data/hf_configs/qwen3-next-80B-A3B_Instruct_FP8_config.json new file mode 100644 index 00000000..b664bd1e --- /dev/null +++ b/vidur-alibabacloud/data/hf_configs/qwen3-next-80B-A3B_Instruct_FP8_config.json @@ -0,0 +1,43 @@ +{ + "architectures": [ + "Qwen3NextForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "decoder_sparse_step": 1, + "eos_token_id": 151645, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 5120, + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_next", + "moe_intermediate_size": 512, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_experts": 512, + "num_experts_per_tok": 10, + "num_hidden_layers": 48, + "num_key_value_heads": 2, + "output_router_logits": false, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000000, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 512, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.57.0.dev0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/vidur-alibabacloud/data/hf_configs/qwen3-next-80B-A3B_config.json b/vidur-alibabacloud/data/hf_configs/qwen3-next-80B-A3B_config.json new file mode 100644 index 00000000..b664bd1e --- /dev/null +++ b/vidur-alibabacloud/data/hf_configs/qwen3-next-80B-A3B_config.json @@ -0,0 +1,43 @@ +{ + "architectures": [ + "Qwen3NextForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "decoder_sparse_step": 1, + "eos_token_id": 151645, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 5120, + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_next", + "moe_intermediate_size": 512, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_experts": 512, + "num_experts_per_tok": 10, + "num_hidden_layers": 48, + "num_key_value_heads": 2, + "output_router_logits": false, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000000, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 512, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.57.0.dev0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh b/vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh new file mode 100644 index 00000000..82d00dc8 --- /dev/null +++ b/vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh @@ -0,0 +1,361 @@ +#!/usr/bin/env bash +# ============================================================================= +# run_scenarios.sh — SimAI / AICB Vidur 四场景一键运行脚本 +# +# 所有文件统一汇聚于 examples/vidur-ali-scenarios/ 目录: +# examples/vidur-ali-scenarios/ +# ├── run_scenarios.sh ← 本脚本 +# ├── logs/ ← tee 运行日志 +# │ └── scenario__.log +# └── simulator_output/ ← vidur 模拟输出 (通过 --output_dir 覆盖) +# └── / +# +# 用法: +# bash examples/vidur-ali-scenarios/run_scenarios.sh --scenario <1|2|3|4> +# bash examples/vidur-ali-scenarios/run_scenarios.sh --all +# bash examples/vidur-ali-scenarios/run_scenarios.sh -h | --help +# +# 场景说明: +# 1 Qwen3-Next-80B 无PD分离 ws=32 (dp=32, tp=1, pp=1, ep=32) 调度: lor +# 2 Qwen3-Next-80B PD分离 ws=8 (P=2, D=6, tp=1, pp=1) 调度: split_wise +# 3 DeepSeek-671B PD分离 ws=8 (P=2, D=6, tp=8, pp=1, ep=8) 调度: split_wise +# 4 Qwen3-MoE-235B PD分离 ws=8 (P=2, D=6, tp=4, pp=1, ep=4) 调度: split_wise +# +# 环境要求: +# conda activate vidur +# conda 路径: /root/miniconda3/envs/vidur +# python: /root/miniconda3/envs/vidur/bin/python +# ============================================================================= + +set -euo pipefail + +# ===================== 路径设置 ===================== +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VIDUR_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +LOG_DIR="$SCRIPT_DIR/logs" +OUTPUT_DIR="$SCRIPT_DIR/simulator_output" + +mkdir -p "$LOG_DIR" +mkdir -p "$OUTPUT_DIR" + +# ===================== 工具函数 ===================== + +cleanup() { + local exit_code=$? + if [[ $exit_code -ne 0 ]]; then + echo "" + echo "[WARN] Script exited abnormally (脚本异常退出), exit_code=$exit_code" + echo " Log dir (日志目录): $LOG_DIR" + echo " Output dir (输出目录): $OUTPUT_DIR" + fi +} +trap 'cleanup' EXIT INT TERM + +validate_environment() { + local conda_env="${CONDA_DEFAULT_ENV:-}" + if [[ "$conda_env" != "vidur" ]]; then + echo "[ERROR] vidur conda env not detected (未检测到 vidur conda 环境)" + echo " Current env (当前环境): ${conda_env:-N/A}" + echo " Please run (请先执行): conda activate vidur" + echo " conda path (路径): /root/miniconda3/envs/vidur" + exit 1 + fi + local python_bin + python_bin="$(which python 2>/dev/null || true)" + if [[ "$python_bin" != */miniconda3/envs/vidur/* ]]; then + echo "[ERROR] python not in vidur env (python 路径不在 vidur 环境内)" + echo " Current python (当前 python): ${python_bin:-not found}" + echo " Expected path (期望路径): /root/miniconda3/envs/vidur/bin/python" + exit 1 + fi + echo "[INFO] Env check passed (环境检查通过): conda=$conda_env, python=$python_bin" +} + +check_disk_space() { + local required_gb=10 + local available + available=$(df "$SCRIPT_DIR" | awk 'NR==2 {print int($4/1024/1024)}') + if [[ "$available" -lt "$required_gb" ]]; then + echo "[ERROR] Insufficient disk space (磁盘空间不足): need ${required_gb}GB, available ${available}GB" + exit 1 + fi + echo "[INFO] Disk check passed (磁盘空间检查通过): available ${available}GB, need ${required_gb}GB" +} + +progress_bar() { + local current=$1 total=$2 + local percent=$((current * 100 / total)) + local filled=$((percent / 5)) + local bar + bar=$(printf "%${filled}s" | tr ' ' '=') + local empty=$((20 - filled)) + local space + space=$(printf "%${empty}s") + printf "\n[%-20s] %d%% (%d/%d)\n" "${bar}${space}" "$percent" "$current" "$total" +} + +validate_scenario_output() { + local scenario_num=$1 + local output_dir=$2 + # Find the latest timestamped output directory + # Use || true to prevent SIGPIPE when ls outputs multiple entries under set -eo pipefail + local latest_dir + latest_dir=$(ls -td "$output_dir"/*/ 2>/dev/null | head -1) || true + if [[ -z "$latest_dir" ]]; then + echo "[WARN] Scenario $scenario_num: no output directory found in $output_dir" + return 1 + fi + if [[ -f "$latest_dir/chrome_trace.json" ]]; then + echo "[INFO] Scenario $scenario_num: validated (chrome_trace.json found)" + else + echo "[WARN] Scenario $scenario_num: chrome_trace.json NOT found in $latest_dir" + return 1 + fi +} + +# ===================== 公共参数 ===================== +# 四个场景共用的硬件/请求生成/后端参数 +COMMON_ARGS=( + # 硬件 + --replica_config_pd_p2p_comm_bandwidth 800 + --replica_config_nvlink_bandwidth 1600 + --replica_config_rdma_bandwidth 800 + --replica_config_pd_p2p_comm_dtype fp8 + --replica_config_network_device h20_dgx + --replica_config_device h20 + # 请求生成: Poisson QPS=100, 固定长度 prefill=100 / decode=8 + --request_generator_config_type synthetic + --interval_generator_config_type poisson + --poisson_request_interval_generator_config_qps 100 + --synthetic_request_generator_config_num_requests 4 + --length_generator_config_type fixed + --fixed_request_length_generator_config_prefill_tokens 100 + --fixed_request_length_generator_config_decode_tokens 8 + --trace_request_length_generator_config_trace_file \ + ./data/processed_traces/splitwise_conv.csv + # 后端 + --random_forrest_execution_time_predictor_config_backend aicb + # 输出目录 → examples/vidur-ali-scenarios/simulator_output/ + --metrics_config_output_dir "$OUTPUT_DIR" +) + +# ===================== 场景函数 ===================== + +# ----------------------------------------------------------------------- +# 场景 1: Qwen3-Next-80B 无PD分离 +# cluster_config_num_replicas = 32 (即 dp=32) +# ws = tp(1) × pp(1) × dp(32) = 32，ep = ws = 32（自动） +# 调度: global=lor, replica=sarathi +# ----------------------------------------------------------------------- +run_scenario_1() { + local ts + ts="$(date +%Y%m%d_%H%M%S)" + local log_file="$LOG_DIR/scenario_1_${ts}.log" + echo "[INFO] === Scenario 1: Qwen3-Next-80B, no PD, ws=32, lor (场景1: 无PD, ws=32, lor) ===" + echo "[INFO] Log (日志): $log_file" + cd "$VIDUR_ROOT" + set +o pipefail + python -m vidur.main \ + "${COMMON_ARGS[@]}" \ + --cluster_config_num_replicas 32 \ + --replica_config_pd_node_ratio 1 \ + --global_scheduler_config_type lor \ + --replica_scheduler_config_type sarathi \ + --replica_config_model_name qwen3-next-80B \ + --replica_config_tensor_parallel_size 1 \ + --replica_config_num_pipeline_stages 1 \ + 2>&1 | tee "$log_file" + local exit_code=${PIPESTATUS[0]} + set -o pipefail + if [[ $exit_code -ne 0 ]]; then + echo "[ERROR] Scenario 1 failed (exit_code=$exit_code), see: $log_file" + return $exit_code + fi + validate_scenario_output 1 "$OUTPUT_DIR" + echo "[INFO] Scenario 1 done (场景1 完成)" +} + +# ----------------------------------------------------------------------- +# 场景 2: Qwen3-Next-80B PD分离 +# 总 replica=8; num_prefill_replicas=2 → prefill dp=2, decode dp=6 +# prefill: ws = tp(1) × pp(1) × dp(2) = 2，ep = 2 +# decode: ws = tp(1) × pp(1) × dp(6) = 6，ep = 6 +# 调度: global=split_wise, replica=split_wise +# ----------------------------------------------------------------------- +run_scenario_2() { + local ts + ts="$(date +%Y%m%d_%H%M%S)" + local log_file="$LOG_DIR/scenario_2_${ts}.log" + echo "[INFO] === Scenario 2: Qwen3-Next-80B, PD, P=2 D=6, split_wise (场景2: PD分离, P=2 D=6) ===" + echo "[INFO] Log (日志): $log_file" + cd "$VIDUR_ROOT" + set +o pipefail + python -m vidur.main \ + "${COMMON_ARGS[@]}" \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --replica_config_num_prefill_replicas 2 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name qwen3-next-80B \ + --replica_config_tensor_parallel_size 1 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_prefill_tensor_parallel_size 1 \ + --replica_config_prefill_num_pipeline_stages 1 \ + --replica_config_decode_tensor_parallel_size 1 \ + --replica_config_decode_num_pipeline_stages 1 \ + 2>&1 | tee "$log_file" + local exit_code=${PIPESTATUS[0]} + set -o pipefail + if [[ $exit_code -ne 0 ]]; then + echo "[ERROR] Scenario 2 failed (exit_code=$exit_code), see: $log_file" + return $exit_code + fi + validate_scenario_output 2 "$OUTPUT_DIR" + echo "[INFO] Scenario 2 done (场景2 完成)" +} + +# ----------------------------------------------------------------------- +# 场景 3: DeepSeek-671B PD分离 +# 总 replica=8; pd_node_ratio=0.25 → prefill dp=2, decode dp=6 +# ws = tp(8) × pp(1) × dp = 16(P)/48(D)，ep = 8 +# 调度: global=split_wise, replica=split_wise +# ----------------------------------------------------------------------- +run_scenario_3() { + local ts + ts="$(date +%Y%m%d_%H%M%S)" + local log_file="$LOG_DIR/scenario_3_${ts}.log" + echo "[INFO] === Scenario 3: DeepSeek-671B, PD, tp=8, ep=8, split_wise (场景3: PD分离, tp=8, ep=8) ===" + echo "[INFO] Log (日志): $log_file" + cd "$VIDUR_ROOT" + set +o pipefail + python -m vidur.main \ + "${COMMON_ARGS[@]}" \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name deepseek-671B \ + --replica_config_tensor_parallel_size 8 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 8 \ + 2>&1 | tee "$log_file" + local exit_code=${PIPESTATUS[0]} + set -o pipefail + if [[ $exit_code -ne 0 ]]; then + echo "[ERROR] Scenario 3 failed (exit_code=$exit_code), see: $log_file" + return $exit_code + fi + validate_scenario_output 3 "$OUTPUT_DIR" + echo "[INFO] Scenario 3 done (场景3 完成)" +} + +# ----------------------------------------------------------------------- +# 场景 4: Qwen3-MoE-235B PD分离 +# 总 replica=8; pd_node_ratio=0.25 → prefill dp=2, decode dp=6 +# ws = tp(4) × pp(1) × dp = 8(P)/24(D)，ep = 4 +# 调度: global=split_wise, replica=split_wise +# ----------------------------------------------------------------------- +run_scenario_4() { + local ts + ts="$(date +%Y%m%d_%H%M%S)" + local log_file="$LOG_DIR/scenario_4_${ts}.log" + echo "[INFO] === Scenario 4: Qwen3-MoE-235B, PD, tp=4, ep=4, split_wise (场景4: PD分离, tp=4, ep=4) ===" + echo "[INFO] Log (日志): $log_file" + cd "$VIDUR_ROOT" + set +o pipefail + python -m vidur.main \ + "${COMMON_ARGS[@]}" \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name qwen3-moe-235B \ + --replica_config_tensor_parallel_size 4 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 4 \ + 2>&1 | tee "$log_file" + local exit_code=${PIPESTATUS[0]} + set -o pipefail + if [[ $exit_code -ne 0 ]]; then + echo "[ERROR] Scenario 4 failed (exit_code=$exit_code), see: $log_file" + return $exit_code + fi + validate_scenario_output 4 "$OUTPUT_DIR" + echo "[INFO] Scenario 4 done (场景4 完成)" +} + +# ===================== 帮助信息 ===================== + +print_help() { + cat <<'EOF' +Usage (用法): + bash examples/vidur-ali-scenarios/run_scenarios.sh --scenario Run single scenario (运行单个场景, N=1~4) + bash examples/vidur-ali-scenarios/run_scenarios.sh --all Run all 4 scenarios (顺序运行全部四个场景) + bash examples/vidur-ali-scenarios/run_scenarios.sh -h | --help Print help (打印帮助) + +Scenarios (场景列表): + 1 Qwen3-Next-80B no PD (无PD分离) ws=32 scheduler: lor + 2 Qwen3-Next-80B PD (PD分离) ws=8 (P=2,D=6) scheduler: split_wise + 3 DeepSeek-671B PD (PD分离) tp=8, ep=8 scheduler: split_wise + 4 Qwen3-MoE-235B PD (PD分离) tp=4, ep=4 scheduler: split_wise + +Output dir (输出目录): examples/vidur-ali-scenarios/simulator_output// +Log dir (日志目录): examples/vidur-ali-scenarios/logs/scenario__.log +EOF +} + +# ===================== 入口 ===================== + +main() { + # --help / -h 不需要环境检查，直接处理 + case "${1:-}" in + -h|--help|"") print_help; exit 0 ;; + esac + + echo "============================================================" + echo " SimAI / AICB Vidur 4-Scenario Runner (四场景运行脚本)" + echo " Root dir (根目录): $SCRIPT_DIR" + echo "============================================================" + + validate_environment + check_disk_space + + case "${1:-}" in + --scenario) + case "${2:-}" in + 1) run_scenario_1 ;; + 2) run_scenario_2 ;; + 3) run_scenario_3 ;; + 4) run_scenario_4 ;; + *) echo "[ERROR] Invalid scenario (无效场景编号): ${2:-}, use 1~4"; exit 1 ;; + esac + ;; + --all) + local total=4 + run_scenario_1 + progress_bar 1 $total + + run_scenario_2 + progress_bar 2 $total + + run_scenario_3 + progress_bar 3 $total + + run_scenario_4 + progress_bar 4 $total + + echo "" + echo "[INFO] All 4 scenarios completed (全部 4 个场景运行完毕)!" + echo " Logs (日志): $LOG_DIR/" + echo " Output (输出): $OUTPUT_DIR/" + ;; + *) + echo "[ERROR] Unknown argument (未知参数): $1" + print_help + exit 1 + ;; + esac +} + +main "$@" diff --git a/vidur-alibabacloud/tests/test_pd_separation.py b/vidur-alibabacloud/tests/test_pd_separation.py new file mode 100644 index 00000000..5fac51f6 --- /dev/null +++ b/vidur-alibabacloud/tests/test_pd_separation.py @@ -0,0 +1,167 @@ +""" +Tests for PD (Prefill-Decode) separation configuration and cluster behavior. + +Covers: + - PD off (pd_node_ratio=1): config init and MIXED mode + - PD on (pd_node_ratio=0.5): cluster creation with independent P/D clusters + - PD params None fallback: prefill_*/decode_* fall back to shared values + - Illegal pd_node_ratio (<=0, >1): must raise ValueError + - num_prefill_replicas priority over pd_node_ratio +""" + +import os +import sys +import pytest + +# Ensure vidur package is importable +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from vidur.config import ( + ClusterConfig, + MetricsConfig, + ReplicaConfig, + SyntheticRequestGeneratorConfig, +) +from vidur.entities.cluster import Cluster + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_cluster( + num_replicas: int = 4, + pd_node_ratio: float = 1.0, + tp: int = 1, + pp: int = 1, + prefill_tp=None, + prefill_pp=None, + decode_tp=None, + decode_pp=None, + num_prefill_replicas=None, + model_name: str = "meta-llama/Llama-2-7b-hf", +) -> Cluster: + """Build a Cluster with the given PD separation params.""" + rc = ReplicaConfig( + model_name=model_name, + tensor_parallel_size=tp, + num_pipeline_stages=pp, + pd_node_ratio=pd_node_ratio, + prefill_tensor_parallel_size=prefill_tp, + prefill_num_pipeline_stages=prefill_pp, + decode_tensor_parallel_size=decode_tp, + decode_num_pipeline_stages=decode_pp, + num_prefill_replicas=num_prefill_replicas, + ) + cc = ClusterConfig(num_replicas=num_replicas, replica_config=rc) + mc = MetricsConfig(write_metrics=False, write_json_trace=False) + gc = SyntheticRequestGeneratorConfig() + return Cluster(cc, mc, gc) + + +# --------------------------------------------------------------------------- +# Test Cases +# --------------------------------------------------------------------------- + +class TestPDOff: + """pd_node_ratio == 1 → MIXED mode, no PD separation.""" + + def test_config_init_defaults(self): + """ReplicaConfig defaults should have pd_node_ratio=1.""" + rc = ReplicaConfig() + assert rc.pd_node_ratio == 1 + assert rc.prefill_tensor_parallel_size is None + assert rc.decode_tensor_parallel_size is None + assert rc.num_prefill_replicas is None + + def test_mixed_mode_cluster(self): + """All replicas are created; prefill_ws == decode_ws.""" + cluster = _make_cluster(num_replicas=4, pd_node_ratio=1, tp=1, pp=1) + rc = cluster._config.replica_config + + assert len(cluster.replicas) == 4 + assert rc.prefill_world_size == rc.decode_world_size + assert rc._num_prefill_replicas == 4 + assert rc._num_decode_replicas == 4 + + +class TestPDOn: + """0 < pd_node_ratio < 1 → PD separation enabled.""" + + def test_pd_cluster_creation(self): + """Replicas are created; P/D counts derived from ratio.""" + cluster = _make_cluster(num_replicas=4, pd_node_ratio=0.5, tp=1, pp=1) + rc = cluster._config.replica_config + + assert len(cluster.replicas) == 4 + assert rc._num_prefill_replicas == 2 + assert rc._num_decode_replicas == 2 + + def test_per_phase_world_size(self): + """prefill_ws and decode_ws are computed independently.""" + cluster = _make_cluster(num_replicas=4, pd_node_ratio=0.5, tp=1, pp=1) + rc = cluster._config.replica_config + + # ws = tp * pp * dp + assert rc.prefill_world_size == 1 * 1 * 2 # tp=1, pp=1, dp=2 + assert rc.decode_world_size == 1 * 1 * 2 + + +class TestPDParamsFallback: + """PD-specific TP/PP params fall back to shared values when None.""" + + def test_none_fallback(self): + """When prefill_tp/decode_tp are None, use shared values.""" + cluster = _make_cluster( + num_replicas=4, pd_node_ratio=0.5, tp=2, pp=1, + ) + rc = cluster._config.replica_config + + assert rc._prefill_tp == 2 # fallback to tp=2 + assert rc._decode_tp == 2 # fallback to tp=2 + assert rc._prefill_pp == 1 + assert rc._decode_pp == 1 + + def test_explicit_per_phase_params(self): + """When prefill_tp/decode_tp are set explicitly, use them.""" + cluster = _make_cluster( + num_replicas=4, pd_node_ratio=0.5, + tp=2, pp=1, + prefill_tp=4, decode_tp=1, + ) + rc = cluster._config.replica_config + + assert rc._prefill_tp == 4 + assert rc._decode_tp == 1 + + +class TestIllegalPdNodeRatio: + """pd_node_ratio <= 0 or > 1 must raise ValueError.""" + + def test_zero_ratio(self): + with pytest.raises(ValueError, match="Invalid pd_node_ratio"): + _make_cluster(num_replicas=4, pd_node_ratio=0) + + def test_negative_ratio(self): + with pytest.raises(ValueError, match="Invalid pd_node_ratio"): + _make_cluster(num_replicas=4, pd_node_ratio=-0.5) + + def test_ratio_greater_than_one(self): + with pytest.raises(ValueError, match="Invalid pd_node_ratio"): + _make_cluster(num_replicas=4, pd_node_ratio=1.5) + + +class TestNumPrefillReplicasPriority: + """num_prefill_replicas takes priority over pd_node_ratio.""" + + def test_explicit_prefill_replicas(self): + """When num_prefill_replicas is set, it overrides pd_node_ratio calc.""" + cluster = _make_cluster( + num_replicas=8, pd_node_ratio=0.5, + num_prefill_replicas=3, + ) + rc = cluster._config.replica_config + + # Should use explicit count, not 0.5 * 8 = 4 + assert rc._num_prefill_replicas == 3 + assert rc._num_decode_replicas == 5 diff --git a/vidur-alibabacloud/vidur/config/config.py b/vidur-alibabacloud/vidur/config/config.py index 14eb13db..10007218 100644 --- a/vidur-alibabacloud/vidur/config/config.py +++ b/vidur-alibabacloud/vidur/config/config.py @@ -483,22 +483,113 @@ class ReplicaConfig: pd_p2p_comm_dtype: str = field( default='float16', - metadata={"help": "> add: pd_p2p_comm_dtype for pd disaggregation."}, + metadata={ + "help": "Data type for PD disaggregation P2P communication.", + "choices": ['fp8', 'float16', 'float32', 'float64', 'bfloat16', 'int8', 'int16', 'int32', 'int64'], + }, ) - pd_node_ratio: float = field( - default=0.5, - metadata={"help": "Number of p replicas : number of d replicas."}, - ) + # pd_node_ratio: float = field( + # default=0.5, + # metadata={"help": "Number of p replicas : number of d replicas."}, + # ) - # parser.add_argument('--expert_model_parallel_size', type=int, default=1, help='Degree of expert model parallelism.') + # when pd_node_ratio = 1, all replicas are P-nodes; no D-nodes; that means not pd disaggregation + pd_node_ratio: float = field( + default=1, + metadata={"help": "Fraction of replicas allocated as prefill (P) nodes. " + "pd_node_ratio=1 (default): MIXED mode, all replicas handle both prefill " + "and decode; PD-specific params (prefill_*/decode_*) are ignored. " + "0 < pd_node_ratio < 1: PD separation enabled, prefill and decode run " + "as independent clusters with potentially different TP/PP/EP. " + "E.g., 0.5 means P:D = 1:1. Must be in range (0, 1]."}, + ) + + # ============================================================ + # [EP Auto] Auto-computed EP, user doesn't need to specify; user value will be overridden + # Temporary = tp * pp (per-replica), later overridden in cluster.py + # to tp * pp * dp (full cluster world_size) + # [EP Auto] EP 自动计算, 用户无需手动传, 传了也会被覆盖 + # 临时值 = tp * pp (per-replica), 后续在 cluster.py 中 + # 覆盖为 tp * pp * dp (全集群 world_size) + # ============================================================ expert_model_parallel_size: int = field( default=1, - metadata={"help": "Degree of expert model parallelism."}, + metadata={"help": "Degree of expert model parallelism. " + "Auto-computed as cluster world_size (tp*pp*dp) in cluster.py, " + "user-provided value will be overridden."}, + ) + + # ============================================================ + # PD separation specific params (optional, fallback to shared values above if not set) + # Reference vLLM: PD separation runs prefill/decode as completely independent clusters + # with potentially different TP/PP/EP + # PD 分离专用参数 (可选, 未指定时 fallback 到上方共享值) + # 参考 vLLM: PD 分离时 prefill/decode 是完全独立的集群 + # 可以有不同 TP/PP/EP + # ============================================================ + prefill_tensor_parallel_size: Optional[int] = field( + default=None, + metadata={"help": "Prefill-specific TP size. Falls back to tensor_parallel_size if not set."}, + ) + prefill_num_pipeline_stages: Optional[int] = field( + default=None, + metadata={"help": "Prefill-specific PP size. Falls back to num_pipeline_stages if not set."}, ) + decode_tensor_parallel_size: Optional[int] = field( + default=None, + metadata={"help": "Decode-specific TP size. Falls back to tensor_parallel_size if not set."}, + ) + decode_num_pipeline_stages: Optional[int] = field( + default=None, + metadata={"help": "Decode-specific PP size. Falls back to num_pipeline_stages if not set."}, + ) + # Directly specify prefill replica count, takes priority over pd_node_ratio + # More flexible: avoids pd_node_ratio indivisibility issues + # 直接指定 prefill replica 数量, 优先于 pd_node_ratio 计算 + # 更灵活: 避免 pd_node_ratio 不整除的问题 + num_prefill_replicas: Optional[int] = field( + default=None, + metadata={"help": "Directly specify number of prefill replicas. " + "Takes priority over pd_node_ratio when set. " + "num_decode_replicas = total_replicas - num_prefill_replicas."}, + ) + def __post_init__(self): + # Base world_size: per-replica GPU count (excluding dp) + # 基础 world_size: per-replica 的 GPU 数 (不含 dp) self.world_size = self.num_pipeline_stages * self.tensor_parallel_size + + # ============================================================ + # [EP] Temporary = tp * pp (per-replica), excluding dp + # Will be overridden in cluster.py to tp * pp * dp (full cluster world_size) + # Initialization only, printed value is for reference + # [EP] 临时值 = tp * pp (per-replica), 不含 dp + # 后续在 cluster.py 中会被覆盖为 tp * pp * dp (全集群 world_size) + # 这里只是初始化, 打印仅供参考 + # ============================================================ + user_ep = self.expert_model_parallel_size + self.expert_model_parallel_size = self.world_size # Temporary, overridden in cluster.py / 临时值, cluster.py 会覆盖 + if user_ep != 1 and user_ep != self.world_size: + logger.debug(f"[EP] Note: user-provided expert_model_parallel_size={user_ep}, " + f"temporarily set to per-replica ws={self.world_size}, " + f"final value will be overridden in cluster.py as tp*pp*dp") + + # 打印 ReplicaConfig 配置摘要 | Print ReplicaConfig summary + logger.debug(f"[ReplicaConfig] tp={self.tensor_parallel_size}, pp={self.num_pipeline_stages}, " + f"per_replica_ws={self.world_size}, ep(temp)={self.expert_model_parallel_size}, " + f"pd_ratio={self.pd_node_ratio}") + if self.pd_node_ratio < 1: + p_tp = self.prefill_tensor_parallel_size or self.tensor_parallel_size + p_pp = self.prefill_num_pipeline_stages or self.num_pipeline_stages + d_tp = self.decode_tensor_parallel_size or self.tensor_parallel_size + d_pp = self.decode_num_pipeline_stages or self.num_pipeline_stages + logger.debug(f"[ReplicaConfig] PD separation enabled: " + f"prefill(tp={p_tp}, pp={p_pp}), decode(tp={d_tp}, pp={d_pp})") + if self.num_prefill_replicas is not None: + logger.debug(f"[ReplicaConfig] User specified num_prefill_replicas={self.num_prefill_replicas}") + self.model_config: BaseModelConfig = BaseModelConfig.create_from_name( self.model_name ) @@ -641,6 +732,12 @@ class BaseExecutionTimePredictorConfig(BasePolyConfig): default="../astra-sim-alibabacloud/inputs/config/SimAI.conf", metadata={"help": "Path to the simai config file."}, ) + aicb_force_bs1: bool = field( + default=True, + metadata={"help": "AICB safe mode: always use bs=1 for CSV generation. " + "Avoids per_token_group_quant_fp8 CUDA kernel incompatibility " + "with bs>1 on some GPUs (e.g. H20). Set to False to use actual batch size."}, + ) @dataclass diff --git a/vidur-alibabacloud/vidur/config/device_sku_config.py b/vidur-alibabacloud/vidur/config/device_sku_config.py index 5ae5c28d..b4705367 100644 --- a/vidur-alibabacloud/vidur/config/device_sku_config.py +++ b/vidur-alibabacloud/vidur/config/device_sku_config.py @@ -33,6 +33,17 @@ def get_type(): # import pdb; pdb.set_trace() # > return DeviceSKUType.A100 +@dataclass +class H20DeviceSKUConfig(BaseDeviceSKUConfig): + fp16_tflops: int = 148 + fp8_tflops: int = 296 + total_memory_gb: int = 141 + + @staticmethod + def get_type(): + # import pdb; pdb.set_trace() # > + return DeviceSKUType.H20 + @dataclass class H100DeviceSKUConfig(BaseDeviceSKUConfig): @@ -45,9 +56,32 @@ def get_type(): @dataclass class H800DeviceSKUConfig(BaseDeviceSKUConfig): - fp16_tflops: int = 1000 + fp16_tflops: int = 989 + fp8_tflops: int = 1979 total_memory_gb: int = 80 @staticmethod def get_type(): - return DeviceSKUType.H800 \ No newline at end of file + return DeviceSKUType.H800 + +@dataclass +class H200DeviceSKUConfig(BaseDeviceSKUConfig): + fp16_tflops: int = 989 + fp8_tflops: int = 1979 + total_memory_gb: int = 141 + + @staticmethod + def get_type(): + return DeviceSKUType.H200 + + +# GB200 NVL72 +@dataclass +class GB200DeviceSKUConfig(BaseDeviceSKUConfig): + fp16_tflops: int = 2500 + fp8_tflops: int = 5000 + total_memory_gb: int = 192 + + @staticmethod + def get_type(): + return DeviceSKUType.GB200 \ No newline at end of file diff --git a/vidur-alibabacloud/vidur/config/model_config.py b/vidur-alibabacloud/vidur/config/model_config.py index 0e689dcb..c89babc1 100644 --- a/vidur-alibabacloud/vidur/config/model_config.py +++ b/vidur-alibabacloud/vidur/config/model_config.py @@ -30,7 +30,6 @@ class BaseModelConfig(BaseFixedConfig): no_tensor_parallel: bool = False -@dataclass @dataclass class DeepseekV3ModelConfig(BaseModelConfig): # "num_hidden_layers": 61, @@ -115,6 +114,127 @@ class DeepseekV3ModelConfig(BaseModelConfig): @staticmethod def get_name(): return "deepseek-671B" + + + + +@dataclass +class Qwen3Next80BA3BModelConfig(BaseModelConfig): + # architectures: list = field(default_factory=lambda: ["Qwen3NextForCausalLM"]) + architectures: str = "Qwen3NextForCausalLM" + attention_dropout: float = 0.0 + bos_token_id: int = 151643 + decoder_sparse_step: int = 1 + eos_token_id: int = 151645 + full_attention_interval: int = 4 + head_dim: int = 256 + hidden_act: str = "silu" + hidden_size: int = 2048 + initializer_range: float = 0.02 + intermediate_size: int = 5120 + linear_conv_kernel_dim: int = 4 + linear_key_head_dim: int = 128 + linear_num_key_heads: int = 16 + linear_num_value_heads: int = 32 + linear_value_head_dim: int = 128 + max_position_embeddings: int = 262144 + mlp_only_layers: list = field(default_factory=list) + model_type: str = "qwen3_next" + moe_intermediate_size: int = 512 + norm_topk_prob: bool = True + num_attention_heads: int = 16 + num_experts: int = 512 + num_experts_per_tok: int = 10 + num_hidden_layers: int = 48 + num_key_value_heads: int = 2 + output_router_logits: bool = False + partial_rotary_factor: float = 0.25 + rms_norm_eps: float = 1e-06 + rope_scaling: Optional[Dict[str, Any]] = None + rope_theta: float = 10000000 + router_aux_loss_coef: float = 0.001 + shared_expert_intermediate_size: int = 512 + tie_word_embeddings: bool = False + torch_dtype: str = "bfloat16" + transformers_version: str = "4.57.0.dev0" + use_cache: bool = True + use_sliding_window: bool = False + vocab_size: int = 151936 + # Fields mapped from base class parameters / 与基类参数对应的字段 + num_layers: int = 48 # maps to num_hidden_layers / 对应 num_hidden_layers + num_q_heads: int = 16 # maps to num_attention_heads / 对应 num_attention_heads + num_kv_heads: int = 2 # maps to num_key_value_heads / 对应 num_key_value_heads + embedding_dim: int = 2048 # maps to hidden_size / 对应 hidden_size + mlp_hidden_dim: int = 5120 # maps to intermediate_size / 对应 intermediate_size + use_gated_mlp: bool = True # per model arch / 根据模型架构设定 + use_bias: bool = False # per model arch / 根据模型架构设定 + use_qkv_bias: bool = False # per model arch / 根据模型架构设定 + activation: ActivationType = ActivationType.SILU # maps to hidden_act / 对应 hidden_act + norm: NormType = NormType.RMS_NORM # per model arch / 根据模型架构设定 + post_attn_norm: bool = True # per model arch / 根据模型架构设定 + + + @staticmethod + def get_name(): + return "qwen3-next-80B" + + +@dataclass +class Qwen3Moe235BA22BModelConfig(BaseModelConfig): + # architectures: list = field(default_factory=lambda: ["Qwen3MoeForCausalLM"]) + architectures: str = "Qwen3MoeForCausalLM" + attention_bias: bool = False + attention_dropout: float = 0.0 + bos_token_id: int = 151643 + decoder_sparse_step: int = 1 + eos_token_id: int = 151645 + head_dim: int = 128 + hidden_act: str = "silu" + hidden_size: int = 4096 + initializer_range: float = 0.02 + intermediate_size: int = 12288 + max_position_embeddings: int = 262144 + max_window_layers: int = 94 + mlp_only_layers: list = field(default_factory=list) + model_type: str = "qwen3_moe" + moe_intermediate_size: int = 1536 + norm_topk_prob: bool = True + num_attention_heads: int = 64 + num_experts: int = 128 + num_experts_per_tok: int = 8 + num_hidden_layers: int = 94 + num_key_value_heads: int = 4 + output_router_logits: bool = False + rms_norm_eps: float = 1e-06 + rope_scaling: Optional[Dict[str, Any]] = None + rope_theta: float = 5000000 + router_aux_loss_coef: float = 0.001 + sliding_window: Optional[int] = None + tie_word_embeddings: bool = False + torch_dtype: str = "bfloat16" + transformers_version: str = "4.51.0" + use_cache: bool = True + use_sliding_window: bool = False + vocab_size: int = 151936 + # Fields mapped from base class parameters / 与基类参数对应的字段 + num_layers: int = 94 # maps to num_hidden_layers / 对应 num_hidden_layers + num_q_heads: int = 64 # maps to num_attention_heads / 对应 num_attention_heads + num_kv_heads: int = 4 # maps to num_key_value_heads / 对应 num_key_value_heads + embedding_dim: int = 4096 # maps to hidden_size / 对应 hidden_size + mlp_hidden_dim: int = 12288 # maps to intermediate_size / 对应 intermediate_size + use_gated_mlp: bool = True # per model arch / 根据模型架构设定 + use_bias: bool = False # per model arch / 根据模型架构设定 + use_qkv_bias: bool = False # per model arch / 根据模型架构设定 + activation: ActivationType = ActivationType.SILU # maps to hidden_act / 对应 hidden_act + norm: NormType = NormType.RMS_NORM # per model arch / 根据模型架构设定 + post_attn_norm: bool = True # per model arch / 根据模型架构设定 + + + + @staticmethod + def get_name(): + # return "qwen3-235B-A22B" + return "qwen3-moe-235B" @dataclass diff --git a/vidur-alibabacloud/vidur/config/node_sku_config.py b/vidur-alibabacloud/vidur/config/node_sku_config.py index e4a8f8c6..6f89c86b 100644 --- a/vidur-alibabacloud/vidur/config/node_sku_config.py +++ b/vidur-alibabacloud/vidur/config/node_sku_config.py @@ -68,4 +68,13 @@ class H800DgxNodeSKUConfig(BaseNodeSKUConfig): @staticmethod def get_type(): - return NodeSKUType.H800_DGX \ No newline at end of file + return NodeSKUType.H800_DGX + +@dataclass +class H20DgxNodeSKUConfig(BaseNodeSKUConfig): + device_sku_type: DeviceSKUType = DeviceSKUType.H20 + num_devices_per_node: int = 8 + + @staticmethod + def get_type(): + return NodeSKUType.H20_DGX \ No newline at end of file diff --git a/vidur-alibabacloud/vidur/entities/batch.py b/vidur-alibabacloud/vidur/entities/batch.py index fea431e0..712f2dd5 100644 --- a/vidur-alibabacloud/vidur/entities/batch.py +++ b/vidur-alibabacloud/vidur/entities/batch.py @@ -52,7 +52,6 @@ def __init__( self._completed_at = None self._scheduled = False self._completed = False - # assert len(requests) <=1, f"> debug: 看看requests的长度是不是始终为1" @property def replica_id(self) -> int: diff --git a/vidur-alibabacloud/vidur/entities/cluster.py b/vidur-alibabacloud/vidur/entities/cluster.py index dba3fe98..e3b5f59b 100644 --- a/vidur-alibabacloud/vidur/entities/cluster.py +++ b/vidur-alibabacloud/vidur/entities/cluster.py @@ -10,12 +10,22 @@ # Cluster contains multiple Replicas class Cluster(BaseEntity): + def __init__( self, cluster_config: ClusterConfig, metrics_config: MetricsConfig, generator_config: BaseRequestGeneratorConfig, ) -> None: + """ + Initialize cluster with replicas based on PD disaggregation config. + 根据 PD 分离配置初始化集群及其 replica + + - pd_node_ratio == 1: MIXED mode, all replicas handle both prefill & decode + pd_node_ratio == 1: MIXED 模式，所有 replica 同时处理 prefill 和 decode + - 0 < pd_node_ratio < 1: PD separation, independent prefill/decode clusters + 0 < pd_node_ratio < 1: PD 分离，独立的 prefill/decode 集群 + """ # >: test when cluster is registered self._id = Cluster.generate_id() @@ -26,13 +36,139 @@ def __init__( # Init replica object handles self._replicas = {} + + rc = self._config.replica_config # shorthand + num_replicas = self._config.num_replicas + + # ============================================================ + # PD disaggregation OFF (pd_node_ratio == 1): all replicas are MIXED type + # PD 分离关闭: 所有 replica 都是 MIXED 类型 + # Each replica handles both prefill and decode + # 每个 replica 同时处理 prefill 和 decode + # EP = ws = tp * pp * dp (full cluster world_size) + # ============================================================ + if rc.pd_node_ratio == 1: + dp = num_replicas + full_ws = rc.tensor_parallel_size * rc.num_pipeline_stages * dp + # [EP Auto] Final EP = full cluster world_size + # [EP Auto] 最终 EP = 全集群 world_size + rc.expert_model_parallel_size = full_ws + + # [Key] Set per-phase attributes in non-PD mode for uniform interface + # [关键] 非PD模式也设置 per-phase 属性, 与PD模式保持统一接口 + # In non-PD mode all replicas handle prefill/decode, sharing the same world_size + # 非PD时所有 replica 同时处理 prefill/decode, 共享同一个 world_size + rc.prefill_world_size = full_ws + rc.decode_world_size = full_ws + rc.prefill_ep = full_ws + rc.decode_ep = full_ws + rc._num_prefill_replicas = num_replicas # All replicas do prefill / 所有 replica 都做 prefill + rc._num_decode_replicas = num_replicas # All replicas do decode / 所有 replica 都做 decode + rc._prefill_tp = rc.tensor_parallel_size + rc._prefill_pp = rc.num_pipeline_stages + rc._decode_tp = rc.tensor_parallel_size + rc._decode_pp = rc.num_pipeline_stages + + logger.debug(f"{'='*70}") + logger.debug(f"[Cluster] PD off, MIXED mode (pd_node_ratio=1)") + logger.debug(f"[Cluster] tp={rc.tensor_parallel_size}, pp={rc.num_pipeline_stages}, " + f"dp={dp}, ws={full_ws}, ep={full_ws}") + logger.debug(f"[Cluster] prefill_ws={rc.prefill_world_size}, decode_ws={rc.decode_world_size} (same)") + logger.debug(f"{'='*70}") + + for _ in range(num_replicas): + replica = Replica(rc, generator_config) + self._replicas[replica.id] = replica + + # ============================================================ + # PD disaggregation ON (0 < pd_node_ratio < 1) + # PD 分离开启 + # Prefill/Decode are independent clusters, may have different TP/PP/EP + # Prefill/Decode 是独立集群, 可有不同 TP/PP/EP + # + # Replica count priority / replica 数量确定优先级: + # 1. num_prefill_replicas (user specified, most flexible) + # num_prefill_replicas (用户直接指定, 最灵活) + # 2. pd_node_ratio (calculated by ratio) + # pd_node_ratio (按比例计算) + # ============================================================ + elif rc.pd_node_ratio > 0 and rc.pd_node_ratio < 1: + # --- Replica count allocation --- + # --- replica 数量分配 --- + if rc.num_prefill_replicas is not None: + # User specified prefill replica count + # 用户直接指定 prefill replica 数量 + num_p = rc.num_prefill_replicas + num_d = num_replicas - num_p + replica_source = f"num_prefill_replicas={rc.num_prefill_replicas} (user specified, 用户指定)" + else: + # Calculate from pd_node_ratio / 通过 pd_node_ratio 计算 + num_p = int(num_replicas * rc.pd_node_ratio) + num_d = num_replicas - num_p + replica_source = f"pd_node_ratio={rc.pd_node_ratio} (by ratio, 按比例)" + + rc._num_prefill_replicas = num_p + rc._num_decode_replicas = num_d + if num_p <= 0 or num_d <= 0: + raise ValueError( + f"[Cluster] _num_prefill_replicas={num_p} and " + f"_num_decode_replicas={num_d} must both be > 0, " + f"source: {replica_source}") + + # --- per-phase TP/PP (fallback to shared values) --- + # --- per-phase TP/PP (回退到共享值) --- + p_tp = rc.prefill_tensor_parallel_size or rc.tensor_parallel_size + p_pp = rc.prefill_num_pipeline_stages or rc.num_pipeline_stages + d_tp = rc.decode_tensor_parallel_size or rc.tensor_parallel_size + d_pp = rc.decode_num_pipeline_stages or rc.num_pipeline_stages + + # --- per-phase world_size and EP --- + # --- per-phase world_size 和 EP --- + # EP = world_size = tp * pp * dp (ref vLLM: EP_SIZE = TP_SIZE x DP_SIZE) + # EP = world_size = tp * pp * dp (参考 vLLM) + rc.prefill_world_size = p_tp * p_pp * num_p + rc.decode_world_size = d_tp * d_pp * num_d + rc.prefill_ep = rc.prefill_world_size + rc.decode_ep = rc.decode_world_size + + # Save per-phase actual TP/PP for later use + # 保存 per-phase 的实际 TP/PP, 方便后续使用 + rc._prefill_tp = p_tp + rc._prefill_pp = p_pp + rc._decode_tp = d_tp + rc._decode_pp = d_pp + + if rc.prefill_world_size <= 0 or rc.decode_world_size <= 0: + raise ValueError( + f"[Cluster] prefill_ws={rc.prefill_world_size} and " + f"decode_ws={rc.decode_world_size} must both be > 0") + + # --- Verbose PD config printout --- + # --- 详尽打印 PD 配置 --- + logger.debug(f"{'='*70}") + logger.debug(f"[PD Config] PD enabled ({replica_source})") + logger.debug(f"[PD Config] Total replicas: {num_replicas} " + f"(prefill={num_p}, decode={num_d})") + logger.debug(f"[PD Config] Prefill: tp={p_tp}, pp={p_pp}, dp={num_p}, " + f"ws={rc.prefill_world_size}, ep={rc.prefill_ep}") + logger.debug(f"[PD Config] Decode: tp={d_tp}, pp={d_pp}, dp={num_d}, " + f"ws={rc.decode_world_size}, ep={rc.decode_ep}") + logger.debug(f"{'='*70}") + + for _ in range(num_replicas): + replica = Replica(rc, generator_config) + self._replicas[replica.id] = replica - for _ in range(self._config.num_replicas): - replica = Replica(self._config.replica_config, generator_config) - self._replicas[replica.id] = replica + else: + raise ValueError( + f"[Cluster] Invalid pd_node_ratio={rc.pd_node_ratio}. " + f"Must be in range (0, 1]. " + f"Use 1 for MIXED mode, or (0, 1) for PD separation." + ) if metrics_config.write_json_trace: self._write_cluster_info_to_file() + @property def replicas(self): diff --git a/vidur-alibabacloud/vidur/entities/execution_time.py b/vidur-alibabacloud/vidur/entities/execution_time.py index b8842ba7..746e4a11 100644 --- a/vidur-alibabacloud/vidur/entities/execution_time.py +++ b/vidur-alibabacloud/vidur/entities/execution_time.py @@ -1,19 +1,419 @@ from vidur.entities.base_entity import BaseEntity - -# > add from vidur.config import ( BaseExecutionTimePredictorConfig, BaseReplicaSchedulerConfig, MetricsConfig, ReplicaConfig, ) +from vidur.logger import init_logger + import os import sys import subprocess +import json +import time as time_module from pathlib import Path import csv -from typing import Dict, Optional -# import Dictionary +from typing import Dict, Optional, Tuple + +logger = init_logger(__name__) + +# 获取当前文件目录，用于计算 aicb 的绝对路径 +# Get current file directory for calculating absolute path to aicb +_CURRENT_FILE_DIR = Path(__file__).resolve().parent +# execution_time.py is under vidur-alibabacloud/vidur/entities/ +# aicb is under workspace_root/aicb/ +# Path: entities/ -> vidur/ -> vidur-alibabacloud/ -> workspace_root/ -> aicb/ +_AICB_ROOT = _CURRENT_FILE_DIR.parent.parent.parent / "aicb" + + +# ============================================================ +# [AICB Optimization B+C] Global Cache + Linear Interpolation +# [AICB优化 B+C方案] 全局缓存 + 首尾插值 +# +# Plan C: Global lookup - avoid repeated AICB CSV reads/runs +# Plan B: Head-tail token strategy - linear interpolation for intermediate seq values +# 方案C: 全局查表 - 避免重复读取/运行AICB +# 方案B: 首尾token策略 - 对中间seq值线性插值 +# +# Cache key: (model_name, ws, tp, pp, ep, bs, seq, phase) +# Cache value: {layer_id: {layer_name: {comp_time, comm_size}}} +# ============================================================ + +# AICB cache data directory +# AICB缓存数据存放目录 +_AICB_WORKLOAD_DIR = _CURRENT_FILE_DIR.parent.parent / "data" / "aicb_workload" +_AICB_CACHE_DIR = _AICB_WORKLOAD_DIR / "cache" +_AICB_LOG_DIR = _AICB_WORKLOAD_DIR / "logs" + + +class AICBGlobalCache: + """ + [AICB Optimization B+C] Global AICB Data Cache + [AICB优化 B+C方案] 全局AICB数据缓存 + + Features / 功能: + 1. Plan C (Lookup): Cache loaded AICB data to avoid repeated CSV reads and subprocess calls + 方案C (查表): 缓存已加载的AICB数据，避免重复CSV读取和subprocess调用 + 2. Plan B (Interpolation): Use head-tail token linear interpolation for unmatched seq values + 方案B (插值): 对于没有精确匹配的seq值，使用首尾token线性插值 + 3. Persistence: Save cache index and data to disk for cross-run reuse + 持久化: 将缓存索引和数据保存到磁盘，跨运行复用 + 4. Logging: Record all AICB calls and cache hit stats + 日志: 记录所有AICB调用和缓存命中情况 + """ + + def __init__(self): + # Core cache: (model, ws, tp, pp, ep, bs, seq, phase) -> parsed data + # 核心缓存 + self._cache: Dict[Tuple, Dict] = {} + + # Statistics counters / 统计计数器 + self._stats = { + 'cache_hits': 0, # Exact cache hits / 精确命中查表 + 'interpolations': 0, # Interpolation hits / 插值命中 + 'aicb_calls': 0, # Actual AICB subprocess calls / 实际AICB subprocess调用 + 'csv_loads': 0, # CSV file load count / CSV文件加载次数 + } + + # Ensure directories exist / 确保目录存在 + _AICB_CACHE_DIR.mkdir(parents=True, exist_ok=True) + _AICB_LOG_DIR.mkdir(parents=True, exist_ok=True) + + # Log file / 日志文件 + self._log_file = _AICB_LOG_DIR / "aicb_cache_log.txt" + + # Load persistent index / 加载持久化索引 + self._index_file = _AICB_WORKLOAD_DIR / "cache_index.json" + self._load_index() + + self._log(f"AICBGlobalCache 初始化完成, 缓存目录: {_AICB_CACHE_DIR}") + + def _log(self, msg: str) -> None: + """Write to log file and print / 写入日志文件并打印""" + timestamp = time_module.strftime("%Y-%m-%d %H:%M:%S") + log_line = f"[{timestamp}] {msg}" + try: + with open(self._log_file, 'a') as f: + f.write(log_line + '\n') + except: + pass + logger.debug(f"[AICB Cache] {msg}") + + def _make_key(self, model_name, ws, tp, pp, ep, bs, seq, phase) -> Tuple: + """Generate cache key / 生成缓存key""" + return (model_name, ws, tp, pp, ep, bs, seq, phase) + + def _make_group_key(self, model_name, ws, tp, pp, ep, bs, phase) -> Tuple: + """Generate group key without seq, for finding seq values to interpolate + 生成不含seq的分组key，用于查找同组的seq值进行插值""" + return (model_name, ws, tp, pp, ep, bs, phase) + + def get(self, model_name, ws, tp, pp, ep, bs, seq, phase) -> Optional[Dict]: + """ + Retrieve AICB data from cache. + 从缓存获取AICB数据 + + Lookup strategy (Plan B+C): + 查找策略 (B+C方案): + 1. Exact match (Plan C): return cached data directly + 精确匹配 (方案C): 直接返回缓存数据 + 2. Linear interpolation (Plan B): interpolate between two nearest seq values + 线性插值 (方案B): 找同组中seq最近的两个值，线性插值 + 3. Nearest neighbor: return the only neighbor if just one exists + 最近邻: 如果只有一个邻居，直接返回 + 4. Miss: return None + 未命中: 返回None + + Returns: + Dict or None: AICB data, or None if cache miss + """ + key = self._make_key(model_name, ws, tp, pp, ep, bs, seq, phase) + + # === 1. Exact match (Plan C: lookup) === + # === 1. 精确匹配 (方案C: 查表) === + if key in self._cache: + self._stats['cache_hits'] += 1 + self._log(f"[命中] 精确匹配: key={self._format_key(key)}, " + f"总命中={self._stats['cache_hits']}") + return self._cache[key] + + # === 2. Try interpolation (Plan B: head-tail token strategy) === + # === 2. 尝试插值 (方案B: 首尾token策略) === + group_key = self._make_group_key(model_name, ws, tp, pp, ep, bs, phase) + neighbors = self._find_neighbors(group_key, seq) + + if neighbors is not None: + interpolated = neighbors + self._stats['interpolations'] += 1 + # Cache interpolated result to avoid repeated computation + # 缓存插值结果，避免重复计算 + self._cache[key] = interpolated + self._log(f"[插值] seq={seq}, 使用邻居插值, " + f"总插值={self._stats['interpolations']}") + return interpolated + + # === 3. Cache miss === + # === 3. 未命中 === + self._log(f"[未命中] key={self._format_key(key)}") + return None + + def put(self, model_name, ws, tp, pp, ep, bs, seq, phase, data: Dict) -> None: + """Store AICB data into cache / 将AICB数据存入缓存""" + key = self._make_key(model_name, ws, tp, pp, ep, bs, seq, phase) + self._cache[key] = data + self._log(f"[缓存] key={self._format_key(key)}, layers={len(data)}") + + # Persist to disk cache / 保存到磁盘缓存 + self._save_cache_entry(key, data) + + def record_aicb_call(self) -> None: + """Record an actual AICB subprocess call / 记录一次实际的AICB subprocess调用""" + self._stats['aicb_calls'] += 1 + self._log(f"[AICB调用] 第{self._stats['aicb_calls']}次subprocess调用") + + def record_csv_load(self) -> None: + """Record a CSV file load / 记录一次CSV文件加载""" + self._stats['csv_loads'] += 1 + + def _find_neighbors(self, group_key, target_seq) -> Optional[Dict]: + """ + [Plan B core] Find neighbors of target_seq in the same group for interpolation. + [方案B核心] 在同组中查找target_seq的邻居，进行线性插值 + + Strategy / 策略: + - Collect all cached seq values in the same group (model, ws, tp, pp, ep, bs, phase) + 找到同组中所有已缓存的seq值 + - Find nearest seq values on both sides of target_seq + 找到target_seq两侧最近的seq值 + - Two neighbors: linear interpolation / 两个邻居: 线性插值 + - One neighbor: nearest neighbor / 一个邻居: 使用最近邻 + - No neighbors: return None / 没有邻居: 返回None + """ + # Collect all cached seq values in the same group + # 收集同组的所有已缓存seq值 + cached_seqs = {} + for key, data in self._cache.items(): + # key = (model, ws, tp, pp, ep, bs, seq, phase) + key_group = (key[0], key[1], key[2], key[3], key[4], key[5], key[7]) + if key_group == group_key: + cached_seqs[key[6]] = data # key[6] = seq + + if not cached_seqs: + return None + + seq_values = sorted(cached_seqs.keys()) + + # Find neighbors on both sides of target_seq + # 找到target_seq两侧的邻居 + lower_seq = None + upper_seq = None + for s in seq_values: + if s <= target_seq: + lower_seq = s + if s >= target_seq and upper_seq is None: + upper_seq = s + + # Exact match (should not reach here, but just in case) + # 精确匹配（不应该到这里，但安全起见） + if target_seq in cached_seqs: + return cached_seqs[target_seq] + + # Two neighbors: linear interpolation + # 两个邻居: 线性插值 + if lower_seq is not None and upper_seq is not None and lower_seq != upper_seq: + alpha = (target_seq - lower_seq) / (upper_seq - lower_seq) + interpolated = self._interpolate(cached_seqs[lower_seq], cached_seqs[upper_seq], alpha) + self._log(f" 插值: seq={target_seq}, lower={lower_seq}, upper={upper_seq}, " + f"alpha={alpha:.4f}") + return interpolated + + # Only one neighbor: nearest neighbor + # 只有一个邻居: 最近邻 + nearest = lower_seq if lower_seq is not None else upper_seq + if nearest is not None: + self._log(f" 最近邻: seq={target_seq}, nearest={nearest}") + return cached_seqs[nearest] + + return None + + def _interpolate(self, data_low: Dict, data_high: Dict, alpha: float) -> Dict: + """ + [Plan B core] Linear interpolation between two AICB datasets. + [方案B核心] 对两组AICB数据进行线性插值 + + For each layer's metric (comp_time, comm_size): + 对每层的每个指标: + result = data_low * (1-alpha) + data_high * alpha + + Args: + data_low: AICB data at lower seq / seq较小时的AICB数据 + data_high: AICB data at higher seq / seq较大时的AICB数据 + alpha: interpolation coefficient [0, 1] / 插值系数 + """ + result = {} + # Merge layer_ids from both datasets + # 取两个数据共有的layer_id + all_layers = set(data_low.keys()) | set(data_high.keys()) + + for layer_id in all_layers: + result[layer_id] = {} + low_layer = data_low.get(layer_id, {}) + high_layer = data_high.get(layer_id, {}) + + # Merge sub-components (attention, mlp, moe, etc.) + # 取两层共有的子组件 + all_components = set(low_layer.keys()) | set(high_layer.keys()) + + for comp_name in all_components: + low_comp = low_layer.get(comp_name, {'comp_time': 0.0, 'comm_size': 0.0}) + high_comp = high_layer.get(comp_name, {'comp_time': 0.0, 'comm_size': 0.0}) + + result[layer_id][comp_name] = { + 'comp_time': low_comp['comp_time'] * (1 - alpha) + high_comp['comp_time'] * alpha, + 'comm_size': low_comp['comm_size'] * (1 - alpha) + high_comp['comm_size'] * alpha, + } + + return result + + def _format_key(self, key: Tuple) -> str: + """Format key as a human-readable string / 格式化key为可读字符串""" + return (f"model={key[0]}, ws={key[1]}, tp={key[2]}, pp={key[3]}, " + f"ep={key[4]}, bs={key[5]}, seq={key[6]}, phase={key[7]}") + + def _save_cache_entry(self, key: Tuple, data: Dict) -> None: + """Save a cache entry to disk / 将缓存条目保存到磁盘""" + try: + # Encode key info into filename + # 文件名编码key信息 + filename = (f"aicb-{key[0]}-ws{key[1]}-tp{key[2]}-pp{key[3]}" + f"-ep{key[4]}-bs{key[5]}-seq{key[6]}-{key[7]}.json") + filepath = _AICB_CACHE_DIR / filename + + # Convert int keys to str keys (JSON requirement) + # 将int key转为str key (JSON要求) + serializable = {} + for lid, ldata in data.items(): + serializable[str(lid)] = ldata + + with open(filepath, 'w') as f: + json.dump(serializable, f, indent=2) + except Exception as e: + self._log(f"[WARNING] 保存缓存条目失败: {e}") + + def _load_index(self) -> None: + """Load cache index and existing data from disk / 从磁盘加载缓存索引和已有数据""" + try: + if _AICB_CACHE_DIR.exists(): + json_files = list(_AICB_CACHE_DIR.glob("aicb-*.json")) + loaded = 0 + for jf in json_files: + try: + # Parse key from filename + # 从文件名解析key + key = self._parse_filename(jf.name) + if key is None: + continue + + with open(jf, 'r') as f: + raw_data = json.load(f) + + # Restore int keys + # 恢复int key + data = {} + for lid_str, ldata in raw_data.items(): + data[int(lid_str)] = ldata + + self._cache[key] = data + loaded += 1 + except: + continue + + if loaded > 0: + self._log(f"从磁盘加载了 {loaded} 条缓存记录") + except: + pass + + def _parse_filename(self, filename: str) -> Optional[Tuple]: + """Parse key from cache filename / 从缓存文件名解析key""" + try: + # aicb-ModelName-ws32-tp4-pp1-ep4-bs1-seq100-prefill.json + if not filename.startswith("aicb-") or not filename.endswith(".json"): + return None + + name = filename[5:-5] # 去掉 "aicb-" 和 ".json" + parts = name.rsplit('-', 7) # 从右边分割，取最后7个部分 + if len(parts) < 8: + return None + + # Last 7 parts: ws{}, tp{}, pp{}, ep{}, bs{}, seq{}, phase + # 最后7个部分 + model_name = parts[0] + phase = parts[-1] + seq = int(parts[-2].replace('seq', '')) + bs = int(parts[-3].replace('bs', '')) + ep = int(parts[-4].replace('ep', '')) + pp = int(parts[-5].replace('pp', '')) + tp = int(parts[-6].replace('tp', '')) + ws = int(parts[-7].replace('ws', '')) + + return (model_name, ws, tp, pp, ep, bs, seq, phase) + except: + return None + + def print_stats(self) -> None: + """Print cache statistics / 打印缓存统计信息""" + total_queries = (self._stats['cache_hits'] + self._stats['interpolations'] + + self._stats['aicb_calls']) + logger.debug(f"\n{'='*70}") + logger.debug(f"[AICB Cache Stats Report]") + logger.debug(f"{'='*70}") + logger.debug(f" Cache entries: {len(self._cache)}") + logger.debug(f" Exact hits: {self._stats['cache_hits']}") + logger.debug(f" Interpolated hits: {self._stats['interpolations']}") + logger.debug(f" AICB real calls: {self._stats['aicb_calls']}") + logger.debug(f" CSV file loads: {self._stats['csv_loads']}") + if total_queries > 0: + hit_rate = (self._stats['cache_hits'] + self._stats['interpolations']) / total_queries * 100 + logger.debug(f" Cache hit rate: {hit_rate:.1f}%") + logger.debug(f" Cache dir: {_AICB_CACHE_DIR}") + logger.debug(f" Log file: {self._log_file}") + logger.debug(f"{'='*70}\n") + + # Also write to log file / 也写入日志 + self._log(f"统计: hits={self._stats['cache_hits']}, " + f"interp={self._stats['interpolations']}, " + f"calls={self._stats['aicb_calls']}, " + f"entries={len(self._cache)}") + + def save_lookup_table(self) -> None: + """Save complete lookup index to JSON for inspection / 保存完整的查表索引到JSON，方便查看""" + try: + table = {} + for key, data in self._cache.items(): + key_str = self._format_key(key) + table[key_str] = { + 'num_layers': len(data), + 'layer_ids': sorted([int(k) for k in data.keys()]), + } + + table_file = _AICB_WORKLOAD_DIR / "lookup_table.json" + with open(table_file, 'w') as f: + json.dump(table, f, indent=2, ensure_ascii=False) + self._log(f"查表索引已保存到 {table_file}") + except Exception as e: + self._log(f"[WARNING] 保存查表索引失败: {e}") + + +# ============================================================ +# Global singleton: all ExecutionTime objects share the same cache +# 全局单例: 所有 ExecutionTime 对象共享同一个缓存 +# ============================================================ +_GLOBAL_AICB_CACHE = AICBGlobalCache() + +# [首尾插值] 记录预加载失败的key，避免重复尝试 +# Record failed preload keys to avoid repeated attempts +_FAILED_PRELOAD_KEYS = set() class ExecutionTime(BaseEntity): def __init__( @@ -73,25 +473,25 @@ def __init__( self._process_model_outputs_time = process_model_outputs_time self._ray_comm_time = ray_comm_time - # > add - # self._config = predictor_config self._config = predictor_config self._replica_config = replica_config self._model_config = replica_config.model_config self.replica_scheduler_config = replica_scheduler_config + # Cache AICB data to avoid repeated loading # 缓存 AICB 数据，避免重复加载 + # Optional[Dict[str, float]]: can be None or a dict mapping str to float # Optional[Dict[str, float]] 表示这个变量可以是 None 或者是一个键为字符串、值为浮点数的字典。 self._aicb_data: Optional[Dict[str, float]] = None + # Two allreduces in mlp and attention layers are implemented here # mlp和attention中的两次allreduce在这里实现 - # Implementation of two allreduces in mlp and attention layers def _get_mlp_layer_execution_time(self) -> float: assert self._mlp_layer_up_proj_execution_time \ + self._mlp_layer_down_proj_execution_time \ + self._mlp_layer_act_execution_time \ + self._tensor_parallel_communication_time \ - + self._mlp_norm_time > 0, f"> debug" + + self._mlp_norm_time > 0, "MLP layer execution time must be positive" return ( self._mlp_layer_up_proj_execution_time + self._mlp_layer_down_proj_execution_time @@ -108,7 +508,7 @@ def _get_attention_layer_execution_time(self) -> float: + self._attention_decode_execution_time \ + self._attention_prefill_execution_time \ + self._tensor_parallel_communication_time \ - + self._attn_norm_time > 0, f"> debug" + + self._attn_norm_time > 0, "Attention layer execution time must be positive" return ( self._attention_layer_pre_proj_execution_time + self._attention_layer_post_proj_execution_time @@ -124,72 +524,95 @@ def _get_attention_layer_execution_time_from_aicb(self,layer_id) -> float: if self._aicb_data is None: self._aicb_data = self._load_aicb_data() - layer_data = self._aicb_data.get(layer_id, {}).get("attention", {}) - # 单位从ns转换为s - # Convert unit from ns to s + # If AICB data is empty, return a small default to avoid division by zero + # 如果 AICB 数据为空，返回一个小的默认值避免除零 + if not self._aicb_data: + logger.warning("AICB data is empty, using default attention execution time") + return 1e-6 # 1 microsecond as default + + layer_data = self._aicb_data.get(layer_id, {}).get("attention", {}) + + # Convert unit from ns to s / 单侍从ns转换为s attention_comp_time = layer_data.get('comp_time', 0.0) * 1e-9 - # 单位Byte - # Unit: Byte + # Unit: Byte / 单位Byte attention_comm_size = layer_data.get('comm_size', 0.0) - attention_time = attention_comp_time + 0 # TODO attention_comm_time - return attention_time + attention_time = attention_comp_time + return attention_time if attention_time > 0 else 1e-6 # def _get_mlp_layer_execution_time_from_dpsk_and_aiob(self) -> float: # def _get_mlp_layer_execution_time_from_aicb(self) -> float: def _get_mlp_layer_execution_time_from_aicb(self, layer_id) -> float: if self._aicb_data is None: self._aicb_data = self._load_aicb_data() - layer_data = self._aicb_data.get(layer_id, {}).get("mlp", {}) - # 单位从ns转换为s - # Convert unit from ns to s + # If AICB data is empty, return a small default to avoid division by zero + # 如果 AICB 数据为空，返回一个小的默认值避免除零 + if not self._aicb_data: + logger.warning("AICB data is empty, using default MLP execution time") + return 1e-6 # 1 microsecond as default + + layer_data = self._aicb_data.get(layer_id, {}).get("mlp", {}) + + # Convert unit from ns to s / 单侍从ns转换为s mlp_comp_time = layer_data.get('comp_time', 0.0) * 1e-9 - # 单位Byte - # Unit: Byte + # Unit: Byte / 单位Byte mlp_comm_size = layer_data.get('comm_size', 0.0) - mlp_time = mlp_comp_time + 0 # TODO mlp_comm_time - return mlp_time + mlp_time = mlp_comp_time + return mlp_time if mlp_time > 0 else 1e-6 def _get_moe_layer_execution_time_from_aicb(self, layer_id) -> float: if self._aicb_data is None: self._aicb_data = self._load_aicb_data() # return self._aicb_data.get("moe") - # 从数据结构中获取对应的值 + # If AICB data is empty, return a small default to avoid division by zero + # 如果 AICB 数据为空，返回一个小的默认值避免除零 + if not self._aicb_data: + logger.warning("AICB data is empty, using default MoE execution time") + return 1e-6 # 1 microsecond as default + # Get corresponding values from the data structure + # 从数据结构中获取对应的值 layer_data = self._aicb_data.get(layer_id, {}).get("moe", {}) # +comm # return layer_data.get('comp_time', 0.0) - replica_stage = "prefill" #TODO stage + replica_stage = "prefill" # NOTE: hardcoded prefill; decode uses different bandwidth - # 单位从ns转换为s - # Convert unit from ns to s + # Convert unit from ns to s / 单侍从ns转换为s moe_comp_time = layer_data.get('comp_time', 0.0) * 1e-9 - # 单位Byte - # Unit: Byte + # Unit: Byte / 单位Byte moe_comm_size = layer_data.get('comm_size', 0.0) if replica_stage == "prefill": # normal kernel - # Gbps换算成 Byte/s - # Convert Gbps to Byte/s + # Convert Gbps to Byte/s / Gbps换算成 Byte/s cur_bw = self._replica_config.rdma_bandwidth * 1024 * 1024 * 1024 / 8 elif replica_stage == "decode": # low_latency kernel - # Gbps换算成 Byte/s - # Convert Gbps to Byte/s + # Convert Gbps to Byte/s / Gbps换算成 Byte/s cur_bw = self._replica_config.nvlink_bandwidth * 1024 * 1024 * 1024 / 8 moe_comm_time = moe_comm_size / cur_bw # 秒 moe_time = moe_comp_time + moe_comm_time - # print(f"> debug layer_id={layer_id} moe_time={moe_time} us moe_comp_time={moe_comp_time} us moe_comm_time={moe_comm_time}") - return moe_time + return moe_time if moe_time > 0 else 1e-6 def _get_aicb_params(self): + """ + Get AICB invocation parameters. + 获取 AICB 调用参数 + + Automatically reads per-phase TP/PP/WS/EP from replica_config. + These values are set correctly per phase in base_execution_time_predictor.py. + 自动从 replica_config 读取 per-phase 的 TP/PP/WS/EP + 这些值已在 base_execution_time_predictor.py 中按 phase 正确设置 + + Returns: + (model_name, model_json_file, tp, pp, ws, ep, bs, seq, phase) + """ if self._replica_config.model_name == 'deepseek-671B': model_name = "DeepSeek-671B" model_json_file = "./scripts/inference_configs/deepseek_default.json" @@ -200,10 +623,13 @@ def _get_aicb_params(self): model_name = "Qwen3-Next-80B" model_json_file = "./scripts/inference_configs/qwen3_next_default.json" + # [PD-Aware] These values are set correctly per phase in base_execution_time_predictor.py + # [PD-Aware] 这些值已在 base_execution_time_predictor.py 中按 phase 正确设置 tp = self._replica_config.tensor_parallel_size pp = self._replica_config.num_pipeline_stages ws = self._replica_config.world_size - ep = self._replica_config.expert_model_parallel_size + # ep = self._replica_config.expert_model_parallel_size + ep = self._replica_config.expert_model_parallel_size # [EP Auto] = per-phase world_size bs = self._replica_config.batch_size seq = self._replica_config.seq_len phase = self._replica_config.phase @@ -211,105 +637,491 @@ def _get_aicb_params(self): return model_name, model_json_file, tp, pp, ws, ep, bs, seq, phase def _get_aicb_csv_path(self) -> str: - """根据当前配置生成 AICB CSV 的预期路径""" - """Generate expected AICB CSV path based on current configuration""" + """Generate expected AICB CSV path based on current configuration + 根据当前配置生成 AICB CSV 的预期路径""" model_name, _, tp, pp, ws, ep, bs, seq, phase = self._get_aicb_params() - print(f'get aicb csv path: {model_name} world_size{ws}-tp{tp}-pp{pp}-ep{ep}-bs{bs}-seq{seq}-{phase}') + logger.debug(f'get aicb csv path: {model_name} world_size{ws}-tp{tp}-pp{pp}-ep{ep}-bs{bs}-seq{seq}-{phase}') filename = ( f"vidur-{model_name}-world_size{ws}-tp{tp}-pp{pp}-ep{ep}" f"-bs{bs}-seq{seq}-{phase}.csv" ) - return os.path.join("results", "workload", filename) + # Use absolute path based on code file location + # 使用基于代码文件位置的绝对路径 + return str(_AICB_ROOT / "results" / "workload" / filename) def _generate_aicb_csv(self): - # TODO > 加生成的代码 - # TODO > Add generation code - return - model_name,model_json_file, tp, pp, ws, ep, bs, seq, phase = self._get_aicb_params() - cwd="../../../aicb/" - - # TODO sys.executable 这样会使用vidur虚拟环境的python，确保与aicb的协同 - # TODO sys.executable This will use vidur virtual environment's python to ensure coordination with aicb + """Generate AICB CSV file / 生成AICB CSV文件""" + model_name, model_json_file, tp, pp, ws, ep, bs, seq, phase = self._get_aicb_params() + logger.debug(f"_generate_aicb_csv: model_name={model_name} model_json_file={model_json_file} tp={tp} pp={pp} ws={ws} ep={ep} bs={bs} seq={seq} phase={phase}") + # Use absolute path based on code file location + # 使用基于代码文件位置的绝对路径 + cwd = str(_AICB_ROOT) + cwd_path = Path(cwd) + + logger.debug(f'\n{"="*80}') + logger.debug(f'===== AICB CSV Generation Debug Info =====') + logger.debug(f'{"="*80}') + + # Check if AICB directory exists / 检查 AICB 目录是否存在 + if not cwd_path.exists(): + logger.error(f'AICB directory does not exist: {cwd}') + logger.error(f'Please ensure AICB is properly installed') + return False + + logger.debug(f'AICB Root Directory: {cwd_path}') + logger.debug(f'AICB Root exists: {cwd_path.exists()}') + + # Check results/workload directory / 检查 results/workload 目录 + results_dir = cwd_path / "results" / "workload" + logger.debug(f'Results directory: {results_dir}') + logger.debug(f'Results directory exists: {results_dir.exists()}') + + # Create directory if not exists / 如果不存在，创建目录 + if not results_dir.exists(): + logger.debug(f'Creating results directory: {results_dir}') + results_dir.mkdir(parents=True, exist_ok=True) + + # List existing files in results/workload directory + # 列出 results/workload 目录下已有的文件 + if results_dir.exists(): + existing_files = list(results_dir.glob('*.csv')) + logger.debug(f'Existing CSV files in results/workload ({len(existing_files)} files):') + for f in existing_files[:10]: + logger.debug(f' - {f.name}') + if len(existing_files) > 10: + logger.debug(f' ... and {len(existing_files) - 10} more files') + + # Build AICB command / 构建AICB命令 cmd = [ - sys.executable, + sys.executable, "-m", "workload_generator.Vidur_workload_generator", - str(model_name), - str(model_json_file), + model_name, + model_json_file, "--seq_length", str(seq), "--micro_batch", str(bs), "--world_size", str(ws), "--tensor_model_parallel_size", str(tp), "--expert_model_parallel_size", str(ep), "--aiob_enable", - "--phase", str(phase), + "--phase", phase, ] - # pp: cmd += ["--pipeline_model_parallel", str(pp)] + + if pp > 1: + cmd.extend(["--pipeline_model_parallel", str(pp)]) + + # Print command for manual execution / 打印可以手动执行的命令 + cmd_str = " ".join(cmd) + logger.debug(f'\n===== Command Details =====') + logger.debug(f'Working directory: cd {cwd_path}') + logger.debug(f'Full command: {cmd_str}') + logger.debug(f'One-liner: cd {cwd_path} && {cmd_str}') + + # Expected output CSV file path / 预期生成的文件路径 + expected_csv = self._get_aicb_csv_path() + logger.debug(f'Expected output CSV file: {expected_csv}') + + try: + logger.debug(f'\n===== Executing Command =====') + result = subprocess.run(cmd, cwd=cwd_path, capture_output=True, text=True, timeout=300) + + logger.debug(f'Return code: {result.returncode}') + if result.stdout.strip(): + logger.debug(f'STDOUT: {result.stdout.strip()}') + if result.stderr.strip(): + logger.debug(f'STDERR: {result.stderr.strip()}') + + # Check results/workload directory after command execution + # 检查命令执行后 results/workload 目录的变化 + logger.debug(f'===== Post-execution Check =====') + if results_dir.exists(): + new_files = list(results_dir.glob('*.csv')) + logger.debug(f'CSV files after execution ({len(new_files)} files)') + + if os.path.exists(expected_csv): + logger.debug(f'SUCCESS: Expected CSV file was created!') + else: + logger.warning(f'Expected CSV file was NOT created!') + similar_files = list(results_dir.glob(f'*{model_name}*{phase}*.csv')) + if similar_files: + logger.debug(f'Similar files found: {[f.name for f in similar_files]}') + else: + logger.debug(f'No similar files found matching pattern: *{model_name}*{phase}*.csv') + + if result.returncode != 0: + logger.error(f'AICB command failed with return code {result.returncode}') + return False + else: + logger.debug(f'AICB command succeeded (returncode=0)') + return True + + except subprocess.TimeoutExpired: + logger.error('AICB command timed out after 300 seconds') + return False + except Exception as e: + logger.error(f'Failed to run AICB command: {e}', exc_info=True) + return False - cwd_path = Path(cwd) - print(f'[DEBUG] run aicb cmd: {cmd}') - result = subprocess.run(cmd, shell=True, capture_output=True, cwd=cwd_path, text=True) - if result.returncode != 0: - raise RuntimeError(f"Command {cmd} failed with return code {result.returncode}") + + def _generate_or_find_bs1_csv( + self, model_name, ws, tp, pp, ep, bs, seq, phase, original_csv_path + ) -> str: + """ + [AICB Safe Mode] Always use bs=1 to generate or find CSV. + [AICB Safe Mode] 始终使用 bs=1 生成或查找 CSV + + Reason: AICB's per_token_group_quant_fp8 CUDA kernel is incompatible + with bs>1 on some GPUs (e.g. H20), causing "CUDA error: invalid + configuration argument". + 原因: AICB 的 per_token_group_quant_fp8 等 CUDA kernel 在某些 GPU + (如 H20) 上对 bs>1 不兼容 + + Strategy / 策略: + 1. If requested bs=1, generate directly / 如果请求的就是 bs=1, 直接生成 + 2. If bs>1, look for existing bs=1 CSV / 如果 bs>1, 查找已有的 bs=1 CSV + 3. If bs=1 CSV missing, generate it / 如果 bs=1 CSV 不存在, 生成它 + + Args: + model_name: Model name / 模型名 + ws, tp, pp, ep: Parallelism config / 并行配置 + bs: Original requested batch size (may be > 1) + seq: Sequence length / 序列长度 + phase: Stage (prefill/decode) / 阶段 + original_csv_path: Original CSV path (may be bs>1) + + Returns: + Path to found or generated CSV (may be bs=1 CSV) + """ + logger.debug(f'[AICB Safe Mode] CSV不存在: {original_csv_path}') + logger.debug(f'[AICB Safe Mode] 请求参数: model={model_name}, bs={bs}, seq={seq}, phase={phase}') + + # ---- Case 1: bs=1 already, generate directly ---- + # ---- 情况1: 本身就是 bs=1, 直接生成 ---- + if bs == 1: + logger.debug(f'[AICB Safe Mode] bs=1, 直接生成...') + _GLOBAL_AICB_CACHE.record_aicb_call() + if self._generate_aicb_csv(): + if os.path.exists(original_csv_path): + logger.debug(f'[AICB Safe Mode] Successfully generated bs=1 CSV: {original_csv_path}') + return original_csv_path + else: + logger.warning('[AICB Safe Mode] 生成后未找到 CSV (文件名可能不匹配)') + else: + logger.warning('[AICB Safe Mode] bs=1 生成失败') + return original_csv_path + + # ---- Case 2: bs > 1, skip original bs, use bs=1 instead ---- + # ---- 情况2: bs > 1, 跳过原始 bs, 直接使用 bs=1 ---- + logger.debug(f'[AICB Safe Mode] bs={bs} > 1, 跳过原始bs (避免CUDA kernel错误), 使用 bs=1') + + # Temporarily switch to bs=1 to get the bs=1 CSV path + # 临时切换到 bs=1 以获取 bs=1 的 CSV 路径 + original_bs = self._replica_config.batch_size + self._replica_config.batch_size = 1 + bs1_csv_path = self._get_aicb_csv_path() + self._replica_config.batch_size = original_bs # Restore immediately / 立即恢复 + + logger.debug(f'[AICB Safe Mode] bs=1 CSV 路径: {bs1_csv_path}') + + # Check if bs=1 CSV already exists + # 检查 bs=1 CSV 是否已存在 + if os.path.exists(bs1_csv_path): + logger.debug('[AICB Safe Mode] 找到已有 bs=1 CSV (无需生成)') + return bs1_csv_path + + # bs=1 CSV doesn't exist, generate it + # bs=1 CSV 不存在, 生成它 + logger.debug('[AICB Safe Mode] bs=1 CSV 不存在, 开始生成...') + original_bs = self._replica_config.batch_size + self._replica_config.batch_size = 1 + + _GLOBAL_AICB_CACHE.record_aicb_call() + gen_ok = self._generate_aicb_csv() + + self._replica_config.batch_size = original_bs # Restore / 恢复 + + if gen_ok and os.path.exists(bs1_csv_path): + logger.debug(f'[AICB Safe Mode] Successfully generated bs=1 CSV: {bs1_csv_path}') + return bs1_csv_path + else: + logger.warning('[AICB Safe Mode] bs=1 生成失败或未找到 CSV') + return original_csv_path # 返回原路径, 后续 fallback 逻辑会处理 def _load_aicb_data(self) -> Dict[int, Dict[str, Dict[str, float]]]: - """加载 CSV，返回 {layer_id: {layer_name: {comp_time: value, comm_size: value}}}""" - """Load CSV, returning {layer_id: {layer_name: {comp_time: value, comm_size: value}}}""" + """ + [AICB Optimization B+C] Load AICB data, preferring global cache and interpolation. + [AICB优化 B+C方案] 加载AICB数据，优先使用全局缓存和插值 + + Lookup flow / 查找流程: + 1. Check global cache exact match (Plan C: lookup) + 检查全局缓存精确匹配 (方案C: 查表) + 2. Check global cache interpolation (Plan B: head-tail token strategy) + 检查全局缓存插值 (方案B: 首尾token策略) + 3. If both miss, read/generate CSV and cache + 如果都没有，读取/生成CSV并缓存 + + Returns: + {layer_id: {layer_name: {comp_time: value, comm_size: value}}} + """ + global _GLOBAL_AICB_CACHE + if self._aicb_data is not None: return self._aicb_data + # Get current parameters / 获取当前参数 + model_name, _, tp, pp, ws, ep, bs, seq, phase = self._get_aicb_params() + + # === Step 1+2: Try global cache (exact match or interpolation) === + # === 步骤1+2: 尝试从全局缓存获取 (精确匹配 或 插值) === + cached_data = _GLOBAL_AICB_CACHE.get(model_name, ws, tp, pp, ep, bs, seq, phase) + if cached_data is not None: + self._aicb_data = cached_data + # [Head-tail interp] On cache hit, also ensure last_seq is preloaded + # [首尾插值] 缓存命中时，也确保last_seq已预加载 + # So even old cache without last_seq can be supplemented + # 这样即使旧缓存中没有last_seq，也能补充加载 + self._ensure_decode_endpoint_preloaded(phase, seq) + return cached_data + + # === Step 3: Cache miss, need to read/generate CSV === + # === 步骤3: 缓存未命中，需要读取/生成CSV === + print(f"[AICB] Cache miss, loading CSV (缓存未命中，需要加载CSV): " + f"model={model_name}, bs={bs}, seq={seq}, phase={phase}") + csv_path = self._get_aicb_csv_path() - full_csv_path = os.path.join("../../../aicb/results/workload/", csv_path) + full_csv_path = csv_path if not os.path.exists(full_csv_path): + if self._config.aicb_force_bs1: + # [AICB Safe Mode] Always use bs=1 to generate/find CSV + # Reason: AICB's per_token_group_quant_fp8 CUDA kernel + # is incompatible with bs>1 on some GPUs (e.g. H20) + # Strategy: Use bs=1 CSV (per-token time is independent of batch size) + full_csv_path = self._generate_or_find_bs1_csv( + model_name, ws, tp, pp, ep, bs, seq, phase, full_csv_path + ) + else: + # Normal mode: generate CSV with actual batch size + _GLOBAL_AICB_CACHE.record_aicb_call() + self._generate_aicb_csv() - # TODO > 加生成的代码 - # TODO > Add generated code - self._generate_aicb_csv() if not os.path.exists(full_csv_path): - print(f'[DEBUG] still not exists {full_csv_path}') - full_csv_path = '../aicb/results/workload/vidur-DeepSeek-671B-world_size32-tp1-pp1-ep32-bs4-seq4096-decode.csv' + # ============================================================ + # [AICB Fallback] Search for existing CSV of the same model + # in results/workload/ directory + # [AICB Fallback] 在 results/workload/ 目录搜索同模型的已有CSV + # + # Search strategy (by priority) / 搜索策略 (按优先级): + # 1. Same model + same ws + same phase (different bs/seq) + # 同模型 + 同ws + 同phase (不同 bs/seq) + # 2. Same model + same phase (different ws/bs/seq) + # 同模型 + 同phase (不同 ws/bs/seq) + # 3. Any model's fallback CSV + # 任意模型的兖底 CSV + # ============================================================ + import glob + search_dir = os.path.dirname(full_csv_path) + + # Get correct model name (from _get_aicb_params, already proper case) + # 获取正确的模型名 + found_fallback = False + + # Priority 1: same model + same ws + same phase + # 优先级1: 同模型 + 同ws + 同phase + pattern1 = os.path.join(search_dir, + f"vidur-{model_name}-world_size{ws}-tp{tp}-pp{pp}-ep{ep}-bs*-seq*-{phase}.csv") + matches1 = sorted(glob.glob(pattern1)) + if matches1: + full_csv_path = matches1[0] + logger.debug(f'[AICB Fallback] Found same model+ws: {full_csv_path}') + found_fallback = True + + # Priority 2: same model + same phase (any ws/ep) + # 优先级2: 同模型 + 同phase (任意 ws/ep) + if not found_fallback: + pattern2 = os.path.join(search_dir, f"vidur-{model_name}-*-{phase}.csv") + matches2 = sorted(glob.glob(pattern2)) + if matches2: + full_csv_path = matches2[0] + logger.debug(f'[AICB Fallback] Found same model: {full_csv_path}') + found_fallback = True + + # Priority 3: any CSV as fallback + # 优先级3: 任意 CSV 兖底 + if not found_fallback: + all_csvs = sorted(glob.glob(os.path.join(search_dir, "vidur-*.csv"))) + if all_csvs: + full_csv_path = all_csvs[0] + logger.debug(f'[AICB Fallback] Using any available CSV: {full_csv_path}') + found_fallback = True + + if not found_fallback: + logger.error('无法找到任何AICB CSV文件') + return {} + + # === Parse CSV === + # === 解析CSV === + _GLOBAL_AICB_CACHE.record_csv_load() + data = self._parse_aicb_csv(full_csv_path) + + if data: + # Store into global cache / 存入全局缓存 + _GLOBAL_AICB_CACHE.put(model_name, ws, tp, pp, ep, bs, seq, phase, data) + + # Also copy CSV to aicb_workload/cache dir for inspection + # 同时复制CSV到aicb_workload/cache目录，方便查看 + try: + import shutil + cache_csv = _AICB_CACHE_DIR / os.path.basename(full_csv_path) + if not cache_csv.exists(): + shutil.copy2(full_csv_path, cache_csv) + except: + pass + + # ============================================================ + # [Head-tail interp optimization] Preload AICB data for decode's last round + # [首尾插值优化] 预加载decode最后一轮的AICB数据 + # ============================================================ + self._ensure_decode_endpoint_preloaded(phase, seq) + + self._aicb_data = data + return data + def _ensure_decode_endpoint_preloaded(self, phase: str, current_seq: int): + """ + [Head-tail interp] Ensure AICB data for decode's last round is preloaded. + [首尾插值] 确保decode最后一轮的AICB数据已预加载 + + Called from both cache-hit and cache-miss paths. + _preload_decode_endpoint internally checks cache, returns immediately if loaded. + 无论是缓存命中还是缓存未命中路径都会调用此方法。 + + Args: + phase: Current stage ("prefill" or "decode") / 当前阶段 + current_seq: Current iteration's seq value / 当前迭代的seq值 + """ + if phase == "decode" and hasattr(self._replica_config, 'decode_last_seq'): + last_seq = self._replica_config.decode_last_seq + if last_seq is not None and last_seq != current_seq: + self._preload_decode_endpoint(last_seq) - # 解析 CSV：按 layer_id 和 layer_name 分组存储所有数据 - # Parsing CSV: Group and store all data by layer_id and layer_name + + def _preload_decode_endpoint(self, last_seq: int): + """ + [Head-tail interp optimization] Preload AICB data for decode's last round. + [首尾插值优化] 预加载decode最后一轮的AICB数据 + + Purpose: When first loading decode round 1 CSV, also load the last round's + seq CSV so intermediate iterations can get more accurate execution times + via linear interpolation. + 目的: 在首次加载decode第一轮CSV时，同时加载最后一轮的seq对应的CSV + + Principle / 原理: + - KV Cache grows during decode, so computation changes with seq + Transformer推理中，decode阶段的KV Cache随seq增长 + - Using only round 1 data (nearest neighbor) ignores this growth trend + 只用第一轮数据(最近邻)会忽略这种增长趋势 + - Head-tail interpolation captures the linear growth + 首尾两点线性插值可以捕捉这种线性增长 + + Args: + last_seq: Seq_len value of the last decode round / 最后一轮decode的seq_len值 + """ + global _GLOBAL_AICB_CACHE, _FAILED_PRELOAD_KEYS + + model_name, _, tp, pp, ws, ep, bs, _, phase = self._get_aicb_params() + + # Note: Must use exact match check, not get() (which triggers nearest neighbor/interpolation) + # 注意: 必须用精确匹配检查，不能用 get() (它会触发最近邻/插值) + # Otherwise seq=106 would match seq=100 via nearest neighbor, skipping real CSV load + # 否则 seq=106 会被最近邻匹配到 seq=100 的数据，跳过真正的CSV加载 + exact_key = _GLOBAL_AICB_CACHE._make_key(model_name, ws, tp, pp, ep, bs, last_seq, phase) + if exact_key in _GLOBAL_AICB_CACHE._cache: + # Already has real data, no need to preload + # 已有真实数据，无需预加载 + return + + # Avoid repeated attempts on failed preloads + # 避免重复尝试已失败的预加载 + if exact_key in _FAILED_PRELOAD_KEYS: + return + + logger.debug(f"[AICB] Preloading decode last round: " + f"model={model_name}, bs={bs}, seq={last_seq}, phase={phase}") + + # Temporarily modify seq_len to generate corresponding CSV path + # 临时修改seq_len以生成对应的CSV路径 + original_seq = self._replica_config.seq_len + self._replica_config.seq_len = last_seq + + csv_path = self._get_aicb_csv_path() + + if not os.path.exists(csv_path): + if self._config.aicb_force_bs1: + # [AICB Safe Mode] Use bs=1, avoid CUDA kernel compatibility issues + logger.debug(f"[AICB] last_seq CSV not found: {csv_path}") + csv_path = self._generate_or_find_bs1_csv( + model_name, ws, tp, pp, ep, bs, last_seq, phase, csv_path + ) + else: + # Normal mode: generate with actual batch size + _GLOBAL_AICB_CACHE.record_aicb_call() + self._generate_aicb_csv() + + if os.path.exists(csv_path): + _GLOBAL_AICB_CACHE.record_csv_load() + data = self._parse_aicb_csv(csv_path) + if data: + _GLOBAL_AICB_CACHE.put(model_name, ws, tp, pp, ep, bs, last_seq, phase, data) + logger.debug(f"[AICB] Successfully cached last_seq={last_seq}, layers={len(data)}") + + # Copy CSV to cache directory / 复制CSV到缓存目录 + try: + import shutil + cache_csv = _AICB_CACHE_DIR / os.path.basename(csv_path) + if not cache_csv.exists(): + shutil.copy2(csv_path, cache_csv) + except: + pass + else: + logger.warning("[AICB首尾插值] last_seq CSV解析为空") + _FAILED_PRELOAD_KEYS.add(exact_key) + else: + logger.warning(f"[AICB首尾插值] 生成后仍未找到last_seq CSV: {csv_path}") + _FAILED_PRELOAD_KEYS.add(exact_key) + + # Restore original seq_len / 恢复原始seq_len + self._replica_config.seq_len = original_seq + + def _parse_aicb_csv(self, csv_path: str) -> Dict[int, Dict[str, Dict[str, float]]]: + """ + Parse AICB CSV file and return structured data. + 解析AICB CSV文件，返回结构化数据 + + Extracted from _load_aicb_data for clean separation. + 从 _load_aicb_data 中提取的CSV解析逻辑。 + + Returns: + {layer_id: {layer_name: {comp_time: float, comm_size: float}}} + """ data: Dict[int, Dict[str, Dict[str, float]]] = {} try: - with open(full_csv_path, newline='') as f: - # 检查文件内容 - # Check File Content - - # content = f.read(1000) # Read the first 1000 characters - # print(f"> debug Read the first 1000 characters: {repr(content)}") - # f.seek(0) # Reset the file pointer - - # 使用制表符作为分隔符，因为这是TSV文件 - # Use tabs as delimiters because this is a TSV file. + with open(csv_path, newline='') as f: reader = csv.DictReader(f, delimiter='\t') - print(f"> debug CSV列名: {reader.fieldnames}") + logger.debug(f"[AICB优化] CSV列名: {reader.fieldnames}") - # 检查是否正确解析了列名 - # Check if column names were parsed correctly if reader.fieldnames and len(reader.fieldnames) == 1: - # 如果列名没有正确分割，尝试手动分割 - # If column names weren't split correctly, try manual splitting actual_fieldnames = reader.fieldnames[0].split('\t') if 'layer_id' in actual_fieldnames and 'layer_name' in actual_fieldnames: - print("> debug Detected tab-separated column names, reprocessing") f.seek(0) lines = f.readlines() - # 手动解析 - # Manual parsing headers = lines[0].strip().split('\t') - print(f"> debug Parsed column names manually:: {headers}") for line_num, line in enumerate(lines[1:], 1): values = line.strip().split('\t') if len(values) == len(headers): row = dict(zip(headers, values)) - # print(f"> debug Row {row_num} data: {row}") - layer_id = int(row['layer_id']) layer_name = row['layer_name'] comp_time = float(row['comp_time']) @@ -321,27 +1133,15 @@ def _load_aicb_data(self) -> Dict[int, Dict[str, Dict[str, float]]]: 'comp_time': comp_time, 'comm_size': comm_size } - print("> debug Manual parsing completed") else: - print("> debug Failed to parse column names correctly") return {} else: - # 正常的CSV解析流程 - # Normal CSV parsing process for row_num, row in enumerate(reader, 1): - # print(f"> debug Row {row_num} data: {row}") - - # 检查必要的键是否存在 - # Check if required keys exist - if 'layer_id' not in row or 'layer_name' not in row or 'comp_time' not in row or 'comm_size' not in row: - print(f"Warning: Row {row_num} missing required columns, skipping") + if 'layer_id' not in row or 'layer_name' not in row: continue - + layer_id = int(row['layer_id']) layer_name = row['layer_name'] - - # 单位：微秒（根据示例） - # Unit: microseconds (based on example) comp_time = float(row['comp_time']) comm_size = float(row['comm_size']) @@ -352,16 +1152,12 @@ def _load_aicb_data(self) -> Dict[int, Dict[str, Dict[str, float]]]: 'comm_size': comm_size } except Exception as e: - print(f"Error reading CSV file: {e}") - import traceback - traceback.print_exc() + logger.error(f"读取CSV文件失败: {e}", exc_info=True) return {} - self._aicb_data = data - # print(f"> debug Successfully loaded data: {data}") + logger.debug(f"[AICB] Successfully parsed CSV: {len(data)} layers from {csv_path}") return data - def _get_block_execution_time(self) -> float: return ( self._get_attention_layer_execution_time() @@ -375,18 +1171,10 @@ def _get_block_execution_time_by_layer_id(self, layer_id: int = 0) -> float: # 根据模型类型确定使用的层类型 # Determine layer type based on model - # if self._replica_config.model_name in ['qwen3-moe-235B']: - # layer_time = self._get_moe_layer_execution_time_from_aicb(layer_id) - # else: - # layer_time = self._get_mlp_layer_execution_time_from_aicb(layer_id) - - # assert att_time >= 0 and layer_time >= 0, f"> debug" - # return att_time + layer_time - att_time = self._get_attention_layer_execution_time_from_aicb(layer_id) mlp_time = self._get_mlp_layer_execution_time_from_aicb(layer_id) moe_time = self._get_moe_layer_execution_time_from_aicb(layer_id) - assert att_time >=0 and mlp_time>=0 and moe_time >= 0, f"> debug" + assert att_time >= 0 and mlp_time >= 0 and moe_time >= 0, "AICB layer times must be non-negative" return att_time + mlp_time + moe_time else: @@ -498,8 +1286,7 @@ def model_time(self) -> float: # 计算当前 pipeline stage 包含的 layer_id 范围 # Calculate the range of layer_ids included in the current pipeline stage - # > TODO: 找_pipeline_stage_id 在哪， 结合batch id - # > TODO: Find where _pipeline_stage_id is defined and integrate with batch id + # NOTE: PP>1 not yet supported, hardcoded stage 0 if self._replica_config.num_pipeline_stages == 1: self._pipeline_stage_id = 0 start_layer = self._pipeline_stage_id * self._num_layers_per_pipeline_stage diff --git a/vidur-alibabacloud/vidur/entities/replica.py b/vidur-alibabacloud/vidur/entities/replica.py index bfb29ce4..412dafb7 100644 --- a/vidur-alibabacloud/vidur/entities/replica.py +++ b/vidur-alibabacloud/vidur/entities/replica.py @@ -1,4 +1,5 @@ from math import ceil +from typing import Tuple from vidur.config import BaseRequestGeneratorConfig, ReplicaConfig from vidur.entities.base_entity import BaseEntity @@ -15,8 +16,8 @@ class ReplicaType(IntEnum): # Define task type enumeration class, inheriting fr DECODE = 2 # Token task (generation stage) -# Replica是一个模型实体，即一个DP单位 # Replica represents a model entity, which is a Data Parallelism (DP) unit +# Replica是一个模型实体，即一个DP单位 class Replica(BaseEntity): def __init__( self, @@ -39,12 +40,12 @@ def __init__( == 0 ) - # > sw - # TODO > Decouple this from replica, as vidur itself is decoupled from it + # TODO(tianhao909): decouple pending_requests from replica + # TODO(tianhao909): 将 pending_requests 从 replica 中解耦 # self._pending_requests = [] self.pending_requests = [] self._pending_tasks = [] - # > scheduler metadata + # Scheduler metadata / 调度器元数据 # self.sched_memory = self.model.size.total_size # Memory usage from scheduler's perspective self.sched_memory = self._device_config.total_memory_gb self.sched_pending_tokens = 0 # Number of pending tokens from scheduler's perspective @@ -61,6 +62,13 @@ def __init__( self.pd_node_ratio = self._replica_config.pd_node_ratio self.nvlink_bandwidth = self._replica_config.nvlink_bandwidth self.rdma_bandwidth = self._replica_config.rdma_bandwidth + + # New variables: track KV cache memory usage + # 新增变量：跟踪kvcache显存使用情况 + self._allocated_kv_cache_memory = 0 # Allocated KV cache memory (bytes) / 已分配的kvcache显存 + self._max_kv_cache_memory = None # Max KV cache capacity (bytes) / 最大kvcache显存容量 + self._kv_cache_allocation_map = {} # Track per-request KV cache allocation / 跟踪每个请求分配的kvcache大小 + @property def id(self) -> int: @@ -146,7 +154,169 @@ def per_device_flops(self) -> float: @property def pending_tasks(self) -> list: return self._pending_tasks + + def get_kv_cache_per_token(self) -> int: + """ + Calculate per-token KV Cache size (unit: Bytes). + 计算每个token的KV Cache大小 (单位: Bytes) + + Formula / 公式: 2 * num_kv_heads * head_dim * num_layers * bytes_per_element + + Returns: + int: Per-token KV Cache size (Bytes) + """ + # Determine bytes per element / 确定每个元素的字节数 + dtype_to_bytes = { + 'float16': 2, 'bfloat16': 2, + 'float32': 4, 'float64': 8, + 'fp8': 1, 'int8': 1, + 'int16': 2, 'int32': 4, 'int64': 8 + } + bytes_per_element = dtype_to_bytes.get(self.pd_p2p_comm_dtype, 2) + + # KV Cache size per token / KV Cache每 token的大小 + kv_cache_per_token = ( + 2 # K和V两个缓存 + * self.num_kv_heads # KV heads数量 + * self.attention_head_dim # 每个head的维度 + * self.num_layers # 层数 + * bytes_per_element # 每个元素的字节数 + ) + return kv_cache_per_token + + def get_remaining_kv_cache_capacity(self, avg_tokens_per_request=None) -> Tuple[int, int]: + """ + Calculate remaining KV cache memory capacity and how many requests it can serve. + 计算当前副本剩余的kvcache显存容量，以及还能容纳多少个request + + Args: + avg_tokens_per_request: Avg tokens per request (default: max_request_tokens) + 每个请求的平均token数 + + Returns: + (remaining_kv_cache_bytes, remaining_request_capacity) + """ + from vidur.scheduler.utils.memory_planner import MemoryPlanner + memory_planner = MemoryPlanner(self._replica_config, self) + + # ===== 1. Init max KV cache capacity (computed on first call) ===== + # ===== 1. 初始化最大kvcache容量 (首次调用时计算) ===== + if self._max_kv_cache_memory is None: + # Get real KV cache available memory (bytes) from memory_planner + # 直接从 memory_planner 获取真实的 KV cache 可用内存 (bytes) + # Correct calculation: available memory - model parameter memory + # 这是正确的计算: 可用内存 - 模型参数内存 + self._max_kv_cache_memory = memory_planner.get_kv_cache_available_memory() + + # Compute per-request KV cache for display + # 计算每请求 KV cache 用于显示 + kv_cache_per_token = self.get_kv_cache_per_token() + tokens_per_req = avg_tokens_per_request or self.max_request_tokens + kv_cache_per_request = kv_cache_per_token * tokens_per_req + max_requests = int(self._max_kv_cache_memory / kv_cache_per_request) if kv_cache_per_request > 0 else 0 + + logger.info(f"[Replica] KV Cache Capacity Init (KV Cache容量初始化):") + logger.info(f" Total GPU mem (GPU总内存): {self.total_memory_gb:.2f} GB") + logger.info(f" Mem margin (内存保留比例): {self.memory_margin_fraction*100:.1f}%") + logger.info(f" Max KV cache capacity (最大KV cache容量): {self._max_kv_cache_memory/(1024**3):.2f} GB") + logger.info(f" KV cache per token (每token KV cache): {kv_cache_per_token} bytes = {kv_cache_per_token/1024:.2f} KB") + logger.info(f" Avg tokens per req (每请求平均token数): {tokens_per_req}") + logger.info(f" KV cache per req (每请求KV cache): {kv_cache_per_request/(1024**3):.4f} GB") + logger.info(f" Max servable reqs (最大可服务请求数): {max_requests}") + + # ===== 2. Compute remaining KV cache memory ===== + # ===== 2. 计算剩余kvcache显存 ===== + remaining_kv_cache = self._max_kv_cache_memory - self._allocated_kv_cache_memory + + # ===== 3. Compute remaining request capacity ===== + # ===== 3. 计算剩余容量可服务的请求数 ===== + # Unified calculation: per-token KV cache * avg tokens per request + # 使用统一的计算方式 + kv_cache_per_token = self.get_kv_cache_per_token() + tokens_per_req = avg_tokens_per_request or self.max_request_tokens + kv_cache_per_request = kv_cache_per_token * tokens_per_req + + if kv_cache_per_request > 0: + remaining_request_capacity = int(remaining_kv_cache / kv_cache_per_request) + else: + remaining_request_capacity = 0 + + # ===== 4. Print debug info ===== + # ===== 4. 打印调试信息 ===== + logger.debug(f"Remaining KV cache: {remaining_kv_cache / (1024**3):.2f} GB ({remaining_kv_cache / (1024**2):.2f} MB)") + logger.debug(f"Per-request KV cache: {kv_cache_per_request/(1024**3):.4f} GB ({tokens_per_req} tokens)") + logger.debug(f"Remaining request capacity: {remaining_request_capacity}") + + return remaining_kv_cache, remaining_request_capacity + + def release_request_kv_cache_memory(self, request) -> None: + """ + Release KV cache memory occupied by the specified request. + 释放指定request占用的kvcache显存 + """ + # Get KV cache size occupied by this request from allocation map + # 从分配映射中获取这个request占用的kvcache大小 + if request.id in self._kv_cache_allocation_map: + kv_cache_size = self._kv_cache_allocation_map[request.id] + assert kv_cache_size > 0, f"fth debug: request {request.id} kv cache size should be positive" + + # Subtract this request's KV cache from allocated total + # 从已分配的kvcache中减去这个request的占用 + self._allocated_kv_cache_memory = max(0, self._allocated_kv_cache_memory - kv_cache_size) + + # Remove this request from allocation map + # 从分配映射中移除这个请求 + del self._kv_cache_allocation_map[request.id] + + logger.debug(f"Released KV cache for request {request.id}: {kv_cache_size / (1024**3):.2f} GB ({kv_cache_size / (1024**2):.2f} MB)") + else: + logger.warning(f"Request {request.id} not found in KV cache allocation map") + + # def allocate_request_kv_cache_memory(self, request, num_blocks): + # """ + # 为指定request分配kvcache显存，使用类似_allocation_map的方式跟踪每个请求的分配情况 + # 根据分配的块数来计算kvcache大小 + # """ + # # 根据分配的块数计算这个request占用的kvcache大小 + # kv_cache_size = request.estimate_kv_cache_size(num_blocks, self) + def allocate_request_kv_cache_memory(self, request, num_blocks, block_size) -> None: + """ + Allocate KV cache memory for a request, tracking per-request allocation. + 为指定request分配kvcache显存，跟踪每个请求的分配情况 + + Previously num_blocks was passed directly as num_tokens, + causing KV cache tracking to be underestimated by block_size times. + Now correctly converts num_blocks * block_size to num_tokens. + 之前 num_blocks 直接作为 num_tokens 传入，导致跟踪量被低估 block_size 倍。 + + Args: + request: Request object / 请求对象 + num_blocks: Number of allocated memory blocks / 分配的内存块数 + block_size: Tokens per block / 每个块包含的token数 + """ + # Correct conversion: num_tokens = num_blocks * block_size + # 正确转换 + num_tokens = num_blocks * block_size + kv_cache_size = request.estimate_kv_cache_size(num_tokens, self) + logger.debug(f"allocate_request_kv_cache_memory: " + f"req={request.id}, num_blocks={num_blocks}, block_size={block_size}, " + f"num_tokens={num_tokens}, kv_cache_size={kv_cache_size/(1024**2):.2f} MB") + + # Update allocation map / 更新分配映射 + if request.id not in self._kv_cache_allocation_map: + self._kv_cache_allocation_map[request.id] = kv_cache_size + else: + # If already allocated, accumulate (for incremental allocation) + # 如果已有分配，则累加 + self._kv_cache_allocation_map[request.id] += kv_cache_size + + # Increase allocated KV cache / 增加已分配的kvcache + self._allocated_kv_cache_memory += kv_cache_size + + logger.debug(f"Allocated KV cache for request {request.id}: {kv_cache_size / (1024**3):.2f} GB, " + f"total allocated: {self._allocated_kv_cache_memory / (1024**3):.2f} GB") + def to_dict(self) -> dict: return { "id": self.id, @@ -162,7 +332,7 @@ def to_dict(self) -> dict: } - def add_to_pool(self, task): + def add_to_pool(self, task) -> None: """ Add a Task to the request pool. Request pool is ordered by request arrival time. @@ -171,7 +341,7 @@ def add_to_pool(self, task): # bisect.insort(): Uses binary search algorithm to insert element into sorted list, maintaining list's sorted state # self.pending_requests: Target list storing all pending requests # task.request: Request object to be inserted - # key=lambda x: x.arrival_timestamp: Sort key function, sorting by request arrival timestamp + # key=lambda x: x.arrival_timestamp: Sort key function, sorting by request arrival time # lambda x: x.arrival_timestamp is an anonymous function that accepts a parameter x (request object) and returns its arrival_timestamp attribute # This ensures the pending_requests list is always sorted by request arrival time if task.request not in self.pending_requests: # If request is not in current pool diff --git a/vidur-alibabacloud/vidur/entities/request.py b/vidur-alibabacloud/vidur/entities/request.py index d528f14a..46d2e2a0 100644 --- a/vidur-alibabacloud/vidur/entities/request.py +++ b/vidur-alibabacloud/vidur/entities/request.py @@ -3,7 +3,6 @@ from vidur.entities.base_entity import BaseEntity from vidur.logger import init_logger -# > from vidur.entities.task import Task import networkx as nx from vidur.entities.flow import Flow @@ -69,8 +68,7 @@ def __init__( self._num_restarts = 0 - # >: Add DAG property - # self.dag: nx.DiGraph = field(default_factory=nx.DiGraph) + # DAG property for PD separation self.dag = nx.DiGraph() self.node_id = 0 self.nodes = {} @@ -91,7 +89,7 @@ def __init__( self.pd_p2p_bytes_per_token = None self.pd_p2p_comm_dtype = None - # > add: Convenient for obtaining the replica corresponding to decode_replica_id through global_scheduler + # Reference to global_scheduler for obtaining decode replica self.global_scheduler = None @@ -265,15 +263,6 @@ def on_batch_end( # Absolute time self._latest_iteration_completed_at = time - # if self._num_processed_tokens == self.total_tokens: - # print(f"> Debug: ") - - # print(f"> Debug: req on_batch_end Request {self._id} processed {num_tokens_processed} tokens, \ - # total processed {self._num_processed_tokens} tokens.") - # print(f"> Debug: req on_batch_end num_processed_tokens={self._num_processed_tokens}, \ - # total_tokens={self.total_tokens} request_type={self.request_type}") - # print(f"> Debug: req on_batch_end At time={time}, \ - # this request's self._completed_at={self._completed_at} self._completed={self._completed}") assert self._num_processed_tokens <= self.total_tokens @@ -282,40 +271,28 @@ def on_batch_end( if self._num_processed_tokens == self._num_prefill_tokens: self._is_prefill_complete = True - # > self.request_type = RequestType.DECODE # we get one decode token when the prefill processing completes self._num_processed_tokens += 1 - # print(f"> Debug: self._num_processed_tokens += 1 \ - # Request {self._id} processed {num_tokens_processed} tokens, \ - # total processed {self._num_processed_tokens} tokens") # we must record the prefill completion time only in the first time # in the subsequent restarts, we keep adding the previously decoded # tokens to the prefill tokens - that is irrelevant to the original prefill if self._prefill_completed_at == 0: - # > At this point it is absolute time, + # Record absolute time of prefill completion self._prefill_completed_at = time # Here; decode batching # elif self._num_processed_tokens == self._num_prefill_tokens: elif self._num_processed_tokens > self._num_prefill_tokens : - # > - assert self._is_prefill_complete == True, "> debug" - assert self.request_type == RequestType.DECODE, "> debug" - - # we get one decode token when the prefill processing completes - # self._num_processed_tokens += 1 - # print(f"> Debug: Request {self._id} at this point _num_processed_tokens > _num_prefill_tokens, \ - # total processed {self._num_processed_tokens} tokens") + assert self._is_prefill_complete == True, "prefill must be complete at this point" + assert self.request_type == RequestType.DECODE, "request type must be DECODE at this point" elif self._num_processed_tokens < self._num_prefill_tokens: - # print(f"> Debug: Request {self._id} at this point _num_processed_tokens < _num_prefill_tokens, \ - # total processed {self._num_processed_tokens} tokens") pass # check if request is completed @@ -323,18 +300,15 @@ def on_batch_end( self._completed_at = time self._completed = True self.decode_time = self._completed_at - self.prefill_completed_at - assert self.decode_time > 0 and self.decode_time < float("inf") , "> Debug: decode time error" - # print(f"> Debug: At this point the request should end!!, \ - # Request {self._id} completed at {self._completed_at} ") + assert self.decode_time > 0 and self.decode_time < float("inf"), "decode_time must be positive and finite" logger.debug(f"Request {self._id} completed at {self._completed_at}") if self._num_processed_tokens >= self._num_prefill_tokens: - # print(f"> Debug: request ID={self._id} self.decode_arrived_at={self.decode_arrived_at} self.request_type={self.request_type} self.prefill_completed_at={self.prefill_completed_at} self._is_prefill_complete={self._is_prefill_complete}") - # assert self.decode_arrived_at < float("inf") and self.request_type == RequestType.DECODE and self.prefill_completed_at > 0 and self._is_prefill_complete == True, "> debug" - assert self.request_type == RequestType.DECODE and self.prefill_completed_at > 0 and self._is_prefill_complete == True, "> debug" + assert self.request_type == RequestType.DECODE and self.prefill_completed_at > 0 and self._is_prefill_complete == True, \ + "post-prefill request must be DECODE with valid prefill_completed_at" @@ -346,8 +320,7 @@ def on_batch_stage_schedule( if self._latest_stage_completed_at == 0: self._preempted_time = 0 else: - # TODO > fy test each time - # print(f"> Debug: request_id={self._id} time={time} self._latest_stage_completed_a={self._latest_stage_completed_at}") + # TODO: verify preempted_time calculation each iteration self._preempted_time += time - self._latest_stage_completed_at self._preempted = False @@ -402,16 +375,11 @@ def restart(self): self._num_restarts += 1 - # > def create_task(self, task_type, **kwargs): """ Creates a Task and adds it to the DAG. """ - # task = Task.from_type(task_type=task_type, - # node_id=next(self.node_id), - # request=self, - # **kwargs) task = Task.from_type(task_type=task_type, node_id=self.node_id, request=self, @@ -419,19 +387,12 @@ def create_task(self, task_type, **kwargs): self.node_id += 1 self.dag.add_node(task) self.nodes[task.node_id] = task - # print(f"> self.dag={self.dag} self.nodes={self.nodes}") - # print(f"> self.dag={self.dag} ") - # import pdb; pdb.set_trace() # > return task def create_flow(self, flow_type, **kwargs): """ Create a flow and add it to the DAG. """ - # flow = Flow.from_type(flow_type=flow_type, - # node_id=next(self.node_id), # Generate unique node ID - # request=self, - # **kwargs) # Create flow based on flow type flow = Flow.from_type(flow_type=flow_type, node_id=self.node_id, # Generate unique node ID request=self, @@ -441,65 +402,74 @@ def create_flow(self, flow_type, **kwargs): self.nodes[flow.node_id] = flow # Add flow to node dictionary return flow # Return created flow - # > def successors(self, node): """ Returns the next Task or Flow to be executed after node. """ return self.dag.successors(node) - # estimate_kv_cache_size - # def estimate_kv_cache_size(self, num_tokens=None, model=None): def estimate_kv_cache_size(self, num_tokens=None, replica=None): """ - 返回生成num_tokens后的KV缓存大小。 - 需要请求的根节点分配到某个实例上。 - Returns the KV-cache size after generating num_tokens - Requires the Request root node to be allocated on an Instance. + Calculate KV Cache size for the given number of tokens (unit: Bytes). + 计算指定token数量的KV Cache大小 (单位: Bytes) + + KV Cache formula / 公式: + kv_cache_size = 2 (K+V) * num_tokens * num_kv_heads * head_dim * num_layers * bytes_per_element + + Args: + num_tokens: Token count (prefill_tokens + decode_tokens) + replica: Replica instance with model config + + Returns: + int: KV Cache size (Bytes) """ - # if num_tokens is None: # If num_tokens is not specified - # num_tokens = self.generated_tokens # Use the number of generated tokens - # if model is None: # If model is not specified - # # model = self.root_node.instance.model # Use root node's model - # model = self.root_node.replica.model # Use root node's model - - # return 2 * self.batch_size * num_tokens * model.architecture.hidden_size \ - # * model.architecture.num_layers * model.size.dtype_size # Calculate KV cache size - # return 2 * self.batch_size * num_tokens * replica.mlp_hidden_dim \ - # * replica.num_layers * replica.size.dtype_size # Calculate KV cache size - # TODO :p2p > self.batch_size and replica.size.dtype_size from vidur - # Point-to-point communication padding; Global parameters; Comm size/bandwidth; - # TODO Another version of ns3; Support writing a stream in config; For later + # ===== 1. Determine bytes per element (by data type) ===== + # ===== 1. 确定每个元素的字节数 (根据数据类型) ===== + dtype_to_bytes = { + 'float16': 2, 'bfloat16': 2, + 'float32': 4, 'float64': 8, + 'fp8': 1, 'int8': 1, + 'int16': 2, 'int32': 4, 'int64': 8 + } + bytes_per_element = dtype_to_bytes.get(replica.pd_p2p_comm_dtype, 2) # Default 2 bytes / 默认2字节 - if replica.pd_p2p_comm_dtype == 'float16': - pd_p2p_bytes_per_token = 2 - elif replica.pd_p2p_comm_dtype == 'float32': - pd_p2p_bytes_per_token = 4 - elif replica.pd_p2p_comm_dtype == 'float64': - pd_p2p_bytes_per_token = 8 - elif replica.pd_p2p_comm_dtype == 'bfloat16': - pd_p2p_bytes_per_token = 2 - elif replica.pd_p2p_comm_dtype == 'int8': - pd_p2p_bytes_per_token = 1 - elif replica.pd_p2p_comm_dtype == 'int16': - pd_p2p_bytes_per_token = 2 - elif replica.pd_p2p_comm_dtype == 'int32': - pd_p2p_bytes_per_token = 4 - elif replica.pd_p2p_comm_dtype == 'int64': - pd_p2p_bytes_per_token = 8 - - self.pd_p2p_bytes_per_token = pd_p2p_bytes_per_token + # Save to instance for reuse elsewhere + # 保存到实例属性供其他地方使用 + self.pd_p2p_bytes_per_token = bytes_per_element self.pd_p2p_comm_dtype = replica.pd_p2p_comm_dtype - assert self.pd_p2p_bytes_per_token is not None and self.pd_p2p_comm_dtype is not None, "> Debug: PD P2P dtype is not set" + # ===== 2. Get KV Cache related dimensions ===== + # ===== 2. 获取KV Cache相关维度 ===== + # Use correct KV cache dims: num_kv_heads * attention_head_dim + # 使用正确的KV cache维度: num_kv_heads * attention_head_dim + # (NOT mlp_hidden_dim, which is MLP's dimension) + # (而不是mlp_hidden_dim, 那是MLP的维度) + num_kv_heads = replica.num_kv_heads + head_dim = replica.attention_head_dim # embedding_dim // num_q_heads + num_layers = replica.num_layers - # TODO : >: double check this - return 2 * num_tokens * replica.mlp_hidden_dim \ - * replica.num_layers * pd_p2p_bytes_per_token # Calculate KV cache size - + # ===== 3. Calculate KV Cache size ===== + # ===== 3. 计算KV Cache大小 ===== + # Formula: 2(K+V) * num_tokens * num_kv_heads * head_dim * num_layers * bytes_per_element + # 公式同上 + kv_cache_size = ( + 2 # K和V两个缓存 + * num_tokens # token数量 + * num_kv_heads # KV heads数量 + * head_dim # 每个head的维度 + * num_layers # 层数 + * bytes_per_element # 每个元素的字节数 + ) + + # ===== 4. Print debug info (first call only) ===== + # ===== 4. 打印调试信息 (首次调用时) ===== + if not hasattr(self, '_kv_cache_debug_printed'): + logger.debug(f"[KV Cache] params: num_tokens={num_tokens}, num_kv_heads={num_kv_heads}, " + f"head_dim={head_dim}, num_layers={num_layers}, bytes={bytes_per_element}") + logger.debug(f"[KV Cache] result: {kv_cache_size} bytes = {kv_cache_size/(1024**3):.4f} GB") + self._kv_cache_debug_printed = True - # return 2 * num_tokens * replica.mlp_hidden_dim \ - # * replica.num_layers # Calculate KV cache size + return kv_cache_size diff --git a/vidur-alibabacloud/vidur/events/batch_end_event.py b/vidur-alibabacloud/vidur/events/batch_end_event.py index 6d9f7bae..f60ced53 100644 --- a/vidur-alibabacloud/vidur/events/batch_end_event.py +++ b/vidur-alibabacloud/vidur/events/batch_end_event.py @@ -42,8 +42,8 @@ def handle_event( - print(f"> Debug: time={self._time} Generates ReplicaScheduleEvent from event {self._id} {self._event_type}, \ - replica_id={self._replica_id}") + logger.debug(f"time={self._time} Generates ReplicaScheduleEvent from event {self._id} {self._event_type}, " + f"replica_id={self._replica_id}") # replica继续将下一个micro-batch加入pipeline # replica continues to add the next micro-batch to the pipeline @@ -56,7 +56,8 @@ def handle_event( # >: Previous code was native vidur; PD separation is added processing; Without PD separation, it won't enter the following path # Check if Splitwise scheduling policy is used - # TODO 250911 > test if non-PD separation works normally + # TODO(tianhao909): test if non-PD separation works normally + # TODO(tianhao909): 测试非 PD 分离模式是否正常工作 if hasattr(scheduler, '__class__') and scheduler.__class__.__name__ == 'SplitwiseGlobalScheduler': # 对于批次中的每个请求，检查是否需要转移到D副本 # For each request in the batch, check if it needs to be transferred to D replica @@ -76,15 +77,16 @@ def handle_event( # request.request_type = RequestType.DECODE - # TODO: > 在这里添加P2P传输带宽时延开销 - # TODO: > Add P2P transmission bandwidth delay overhead here + # TODO(tianhao909): add P2P transmission bandwidth delay overhead here + # TODO(tianhao909): 在这里添加 P2P 传输带宽时延开销 # transfer_delay = calculate_p2p_transfer_delay(request) # request.decode_arrived_at += transfer_delay # transfer_delay = 1 # > assumption # transfer_delay = 10 # > assumption # request.pd_p2p_comm_size = request.estimate_kv_cache_size() - assert request.num_processed_tokens == request.num_prefill_tokens + 1 , "> debug" + assert request.num_processed_tokens == request.num_prefill_tokens + 1, \ + "processed tokens must equal prefill tokens + 1 at this point" request.pd_p2p_comm_size = request.estimate_kv_cache_size( request.num_processed_tokens, replica_scheduler.replica) # replica_scheduler.replica @@ -92,12 +94,13 @@ def handle_event( # transfer_delay = request.pd_p2p_comm_size / (request.bandwidth - request.bandwidth_used) # transfer_delay = request.pd_p2p_comm_size / request.bandwidth - # TODO >: request.bandwidth 具体怎么赋值， 怎么传，应该是个topo； 或者考虑竞争？ - # TODO >: How exactly is request.bandwidth assigned and passed, should be a topology; Or consider contention? + # TODO(tianhao909): determine bandwidth from topology with contention modeling + # TODO(tianhao909): bandwidth 应该从 topo 获取，并考虑竞争 # request.pd_p2p_comm_bandwidth = 400*1024*1024*1024 request.pd_p2p_comm_bandwidth = replica_scheduler.replica.pd_p2p_comm_bandwidth*1024*1024*1024/8 - assert request.pd_p2p_comm_size < float('inf') and request.pd_p2p_comm_size > 0 and request.pd_p2p_comm_bandwidth > 0 , "> debug" + assert request.pd_p2p_comm_size < float('inf') and request.pd_p2p_comm_size > 0 and request.pd_p2p_comm_bandwidth > 0, \ + "P2P communication size and bandwidth must be valid" request.pd_p2p_comm_time = request.pd_p2p_comm_size / request.pd_p2p_comm_bandwidth @@ -108,17 +111,27 @@ def handle_event( # 从P副本中删除请求 # Remove request from P replica - # TODO: > 250911 写两个req p 和 d的token数目都很少； 测试内存判断的逻辑对不对；整体等逻辑对不对 - # TODO: > 250911 Write two requests with few tokens for both p and d; Test if memory judgment logic is correct; Overall logic correctness + # TODO(tianhao909): write small-token test cases for memory logic validation + # TODO(tianhao909): 写两个 req p 和 d 的 token 数目都很少；测试内存判断的逻辑对不对 # > 隐患 replica 清除 req时候， 对应的内存块也要清除 # > risk: When replica clears requests, corresponding memory blocks should also be cleared p_replica_scheduler = replica_scheduler if request in p_replica_scheduler.replica.pending_requests: + # 在移除请求之前，先计算当前的kvcache使用情况 + # print(f"> 在移除请求 {request.id} 之前:") + # p_replica_scheduler.replica.get_remaining_kv_cache_capacity() + + # 移除请求 p_replica_scheduler.replica.pending_requests.remove(request) - # TODO：> 确保对应的存储也清空了 - # TODO: > Ensure corresponding storage is also cleared + # 移除请求后释放相应的显存 + # p_replica_scheduler.replica.release_request_kv_cache_memory(request) + # print(f"> 请求 {request.id} 已从Prefill副本移除并释放显存") + # p_replica_scheduler.replica.get_remaining_kv_cache_capacity() + + # TODO(tianhao909): ensure corresponding storage is also cleared + # TODO(tianhao909): 确保对应的存储也清空了 # 将请求添加到D副本，获取对应的D副本并添加请求 # Add request to D replica, get corresponding D replica and add request @@ -130,21 +143,150 @@ def handle_event( # Generate D replica scheduling event events.append(ReplicaScheduleEvent(request.decode_arrived_at, request.decode_replica_id)) - print(f"> Debug: pd d-path time={self._time} Generates ReplicaScheduleEvent from event {self._id} {self._event_type}, \ - decode_replica_id={request.decode_replica_id} len(events)={len(events)}") + logger.debug(f"pd d-path time={self._time} Generates ReplicaScheduleEvent from event {self._id} {self._event_type}, " + f"decode_replica_id={request.decode_replica_id} len(events)={len(events)}") if request._num_processed_tokens >= request._num_prefill_tokens: # print(f"> self.decode_arrived_at={self.decode_arrived_at} self.request_type={self.request_type} self.prefill_completed_at={self.prefill_completed_at} self._is_prefill_complete={self._is_prefill_complete}") - assert request.decode_arrived_at < float("inf") and request.request_type == RequestType.DECODE and request.prefill_completed_at > 0 and request._is_prefill_complete == True, "> debug" + assert request.decode_arrived_at < float("inf") and request.request_type == RequestType.DECODE and request.prefill_completed_at > 0 and request._is_prefill_complete == True, \ + "post-prefill request must have valid decode_arrived_at and be in DECODE state" + # Call memory info logging function (disabled) + # 调用显存信息日志函数（已禁用） + # self._log_memory_info(scheduler) return events + def _log_memory_info(self, scheduler: BaseGlobalScheduler) -> None: + """ + Get and print memory capacity info for prefill and decode replicas. + 获取并打印prefill和decode副本的各种显存容量信息 + """ + # Get all replicas from scheduler + # 获取scheduler中所有的replica + # Use scheduler._replica_schedulers to get all replica IDs + # 使用scheduler的_replica_schedulers属性获取所有副本ID + replica_ids = list(scheduler._replica_schedulers.keys()) + + # Separate prefill and decode replica info + # 分别记录prefill和decode副本的信息 + prefill_replica_info = {} + decode_replica_info = {} + + for replica_id in replica_ids: + replica_scheduler = scheduler.get_replica_scheduler(replica_id) + replica = replica_scheduler.replica + + # Get TP and PP parameters / 获取TP和PP参数 + tensor_parallel_size = replica._replica_config.tensor_parallel_size + pipeline_parallel_size = replica._replica_config.num_pipeline_stages + + # Create param_counter from replica_config + # 从replica_config创建param_counter + param_counter = replica._replica_config._param_counter if hasattr(replica._replica_config, '_param_counter') else None + if param_counter is None: + # If replica has no _param_counter, create from replica_config + # 如果replica本身没有_param_counter，尝试从replica_config创建 + from vidur.utils.param_counter import ParamCounter + param_counter = ParamCounter(replica._replica_config) + + # Get model params memory usage / 获取模型参数占用的显存 + total_params = param_counter.get_num_parameters_per_device() + # Convert bytes to GB / 将bytes转换为GB + total_params_gb = total_params / (1024**3) + + # Create memory_planner from replica_config and replica + # 从replica_config和replica创建memory_planner + from vidur.scheduler.utils.memory_planner import MemoryPlanner + memory_planner = MemoryPlanner(replica._replica_config, replica) + + # Get reserved KV cache memory / 获取kvcache预留的显存 + max_batch_size = memory_planner.get_max_batch_size() + kv_cache_per_request = memory_planner._get_kv_cache_memory_per_device_per_request() + memory_for_kv_cache = kv_cache_per_request * max_batch_size + + # Convert bytes to GB / 将bytes转换为GB + memory_for_kv_cache_gb = memory_for_kv_cache / (1024**3) + + # Get actual running requests' KV cache memory + # Note: Replica has no running_requests attr, so we only count pending_requests + # 获取实际运行的request的kvcache显存容量 + # 注意：Replica对象没有running_requests属性，只统计pending_requests + pending_requests_count = len(replica.pending_requests) + actual_kv_cache_memory = kv_cache_per_request * pending_requests_count + actual_kv_cache_memory_gb = actual_kv_cache_memory / (1024**3) + + # Compute whole-replica values (per-GPU * TP * PP) + # 计算整个replica的值（单GPU值 × TP × PP） + total_memory_replica_gb = replica.total_memory_gb * tensor_parallel_size * pipeline_parallel_size + params_memory_replica_gb = total_params_gb * tensor_parallel_size * pipeline_parallel_size + reserved_kv_cache_memory_replica_gb = memory_for_kv_cache_gb * tensor_parallel_size * pipeline_parallel_size + actual_running_kv_cache_memory_replica_gb = actual_kv_cache_memory_gb * tensor_parallel_size * pipeline_parallel_size + + # Store info / 存储信息 + replica_info = { + 'total_memory_gb': replica.total_memory_gb, + 'total_memory_replica_gb': total_memory_replica_gb, + 'params_memory_gb': total_params_gb, + 'params_memory_replica_gb': params_memory_replica_gb, + 'reserved_kv_cache_memory_gb': memory_for_kv_cache_gb, + 'reserved_kv_cache_memory_replica_gb': reserved_kv_cache_memory_replica_gb, + 'actual_running_kv_cache_memory_gb': actual_kv_cache_memory_gb, + 'actual_running_kv_cache_memory_replica_gb': actual_running_kv_cache_memory_replica_gb, + 'active_requests_count': pending_requests_count, + 'max_batch_size': max_batch_size, + 'tp': tensor_parallel_size, + 'pp': pipeline_parallel_size + } + + if replica.replica_type == ReplicaType.PREFILL: + prefill_replica_info[replica_id] = replica_info + elif replica.replica_type == ReplicaType.DECODE: + decode_replica_info[replica_id] = replica_info + + # 打印信息 | Print memory info + logger.info("=" * 100) + logger.info("Memory Usage Statistics (GB) (显存使用情况统计):") + logger.info("-" * 100) + + if prefill_replica_info: + logger.info("Prefill Replica Memory Info (Prefill副本显存信息):") + for pid, info in prefill_replica_info.items(): + logger.info(f" Replica ID {pid} (TP={info['tp']}, PP={info['pp']}):") + logger.info(f" Per-GPU total mem (单GPU总显存容量): {info['total_memory_gb']:.2f} GB ({info['total_memory_gb']*1024:.2f} MB)") + logger.info(f" Replica total mem (整个Replica总显存容量): {info['total_memory_replica_gb']:.2f} GB ({info['total_memory_replica_gb']*1024:.2f} MB)") + logger.info(f" Per-GPU model params (单GPU模型参数占用显存): {info['params_memory_gb']:.2f} GB ({info['params_memory_gb']*1024:.2f} MB)") + logger.info(f" Replica model params (整个Replica模型参数占用显存): {info['params_memory_replica_gb']:.2f} GB ({info['params_memory_replica_gb']*1024:.2f} MB)") + logger.info(f" Per-GPU reserved KV cache (单GPU预留kvcache显存): {info['reserved_kv_cache_memory_gb']:.2f} GB ({info['reserved_kv_cache_memory_gb']*1024:.2f} MB)") + logger.info(f" Replica reserved KV cache (整个Replica预留kvcache显存): {info['reserved_kv_cache_memory_replica_gb']:.2f} GB ({info['reserved_kv_cache_memory_replica_gb']*1024:.2f} MB)") + logger.info(f" Per-GPU actual KV cache (单GPU实际kvcache显存): {info['actual_running_kv_cache_memory_gb']:.2f} GB ({info['actual_running_kv_cache_memory_gb']*1024:.2f} MB)") + logger.info(f" Replica actual KV cache (整个Replica实际kvcache显存): {info['actual_running_kv_cache_memory_replica_gb']:.2f} GB ({info['actual_running_kv_cache_memory_replica_gb']*1024:.2f} MB)") + logger.info(f" Active requests (当前活跃请求数): {info['active_requests_count']}") + logger.info(f" Max batch size (最大批处理大小): {info['max_batch_size']}") + logger.info("-" * 100) + + if decode_replica_info: + logger.info("Decode Replica Memory Info (Decode副本显存信息):") + for did, info in decode_replica_info.items(): + logger.info(f" Replica ID {did} (TP={info['tp']}, PP={info['pp']}):") + logger.info(f" Per-GPU total mem (单GPU总显存容量): {info['total_memory_gb']:.2f} GB ({info['total_memory_gb']*1024:.2f} MB)") + logger.info(f" Replica total mem (整个Replica总显存容量): {info['total_memory_replica_gb']:.2f} GB ({info['total_memory_replica_gb']*1024:.2f} MB)") + logger.info(f" Per-GPU model params (单GPU模型参数占用显存): {info['params_memory_gb']:.2f} GB ({info['params_memory_gb']*1024:.2f} MB)") + logger.info(f" Replica model params (整个Replica模型参数占用显存): {info['params_memory_replica_gb']:.2f} GB ({info['params_memory_replica_gb']*1024:.2f} MB)") + logger.info(f" Per-GPU reserved KV cache (单GPU预留kvcache显存): {info['reserved_kv_cache_memory_gb']:.2f} GB ({info['reserved_kv_cache_memory_gb']*1024:.2f} MB)") + logger.info(f" Replica reserved KV cache (整个Replica预留kvcache显存): {info['reserved_kv_cache_memory_replica_gb']:.2f} GB ({info['reserved_kv_cache_memory_replica_gb']*1024:.2f} MB)") + logger.info(f" Per-GPU actual KV cache (单GPU实际kvcache显存): {info['actual_running_kv_cache_memory_gb']:.2f} GB ({info['actual_running_kv_cache_memory_gb']*1024:.2f} MB)") + logger.info(f" Replica actual KV cache (整个Replica实际kvcache显存): {info['actual_running_kv_cache_memory_replica_gb']:.2f} GB ({info['actual_running_kv_cache_memory_replica_gb']*1024:.2f} MB)") + logger.info(f" Active requests (当前活跃请求数): {info['active_requests_count']}") + logger.info(f" Max batch size (最大批处理大小): {info['max_batch_size']}") + logger.info("-" * 100) + logger.info("=" * 100) + - def to_dict(self): + def to_dict(self) -> dict: return { "time": self.time, "event_type": self.event_type, "batch_id": self._batch.id, - } + } \ No newline at end of file diff --git a/vidur-alibabacloud/vidur/events/batch_stage_arrival_event.py b/vidur-alibabacloud/vidur/events/batch_stage_arrival_event.py index bb27b615..e583678f 100644 --- a/vidur-alibabacloud/vidur/events/batch_stage_arrival_event.py +++ b/vidur-alibabacloud/vidur/events/batch_stage_arrival_event.py @@ -10,6 +10,7 @@ logger = init_logger(__name__) +# A micro-batch arrives at a PP stage # 一个micro-batch到达某个PP stage class BatchStageArrivalEvent(BaseEvent): def __init__(self, time: float, replica_id: int, stage_id: int, batch: Batch): diff --git a/vidur-alibabacloud/vidur/events/batch_stage_end_event.py b/vidur-alibabacloud/vidur/events/batch_stage_end_event.py index 647046c3..ed3954a4 100644 --- a/vidur-alibabacloud/vidur/events/batch_stage_end_event.py +++ b/vidur-alibabacloud/vidur/events/batch_stage_end_event.py @@ -53,16 +53,13 @@ def handle_event( next_events = [ # 当前stage调度下一个micro-batch - # TODO: 这里有点怪，BatchStageEndEvent会引发当前stage的调度 - # BatchStageArrivalEvent也会引发当前stage的调度 - # 虽然多次调度并不会引发问题，但是有很多调度是多余的(因为stage_scheduler.is_busy = True - # 或者stage_scheduler.queue为空) - - # Schedule the next micro-batch on the current stage - # TODO: This seems odd, BatchStageEndEvent triggers scheduling of the current stage - # BatchStageArrivalEvent also triggers scheduling of the current stage - # Although multiple scheduling doesn't cause issues, many schedules are redundant + # TODO(tianhao909): odd behavior - BatchStageEndEvent triggers current stage scheduling + # BatchStageArrivalEvent also triggers current stage scheduling + # Although multiple scheduling doesn't cause issues, many schedules are redundant # (because stage_scheduler.is_busy = True or stage_scheduler.queue is empty) + # TODO(tianhao909): 这里有点怪，BatchStageEndEvent 会触发当前 stage 的调度 + # BatchStageArrivalEvent 也会触发当前 stage 的调度 + # 虽然多次调度不会引发问题，但很多调度是冗余的 ReplicaStageScheduleEvent( self.time, self._replica_id, diff --git a/vidur-alibabacloud/vidur/events/replica_schedule_event.py b/vidur-alibabacloud/vidur/events/replica_schedule_event.py index 208e7687..bd0ed514 100644 --- a/vidur-alibabacloud/vidur/events/replica_schedule_event.py +++ b/vidur-alibabacloud/vidur/events/replica_schedule_event.py @@ -27,9 +27,9 @@ def handle_event( replica_scheduler = scheduler.get_replica_scheduler(self._replica_id) # _batches中至多有PP-stages个batch - # TODO: 这里有一点奇怪，他这样的话就是每次issue PP-stages个batch + # TODO(tianhao909): odd behavior - issues PP-stages batches each time # _batches contains at most PP-stages batches - # TODO: This is a bit strange, as it issues PP-stages batches each time + # TODO(tianhao909): 这里有点奇怪，每次发射 PP-stages 个 batch self._batches = replica_scheduler.on_schedule() if not self._batches: diff --git a/vidur-alibabacloud/vidur/events/replica_stage_schedule_event.py b/vidur-alibabacloud/vidur/events/replica_stage_schedule_event.py index d6a451e9..724eb78d 100644 --- a/vidur-alibabacloud/vidur/events/replica_stage_schedule_event.py +++ b/vidur-alibabacloud/vidur/events/replica_stage_schedule_event.py @@ -47,10 +47,11 @@ def handle_event( self._is_last_stage = stage_scheduler.is_last_stage - print(f"> Debug: time={self._time} Event {self._id} of type {self._event_type} \ - Generates 1 BatchStageEndEvent replica_id={self._replica_id} stage_id={self._stage_id} \ - batch_stage={self._batch_stage}") - assert self._batch_stage.execution_time >= 0, f"> debug self._batch_stage.execution_time={self._batch_stage.execution_time}" + logger.debug(f"time={self._time} Event {self._id} of type {self._event_type} " + f"Generates 1 BatchStageEndEvent replica_id={self._replica_id} stage_id={self._stage_id} " + f"batch_stage={self._batch_stage}") + assert self._batch_stage.execution_time >= 0, \ + f"batch_stage execution_time must be non-negative, got {self._batch_stage.execution_time}" return [ BatchStageEndEvent( diff --git a/vidur-alibabacloud/vidur/execution_time_predictor/base_execution_time_predictor.py b/vidur-alibabacloud/vidur/execution_time_predictor/base_execution_time_predictor.py index d937471d..27bf45f4 100644 --- a/vidur-alibabacloud/vidur/execution_time_predictor/base_execution_time_predictor.py +++ b/vidur-alibabacloud/vidur/execution_time_predictor/base_execution_time_predictor.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from vidur.execution_time_predictor.communication_time_predictor import TPTimePredictor +from vidur.logger import init_logger from vidur.config import ( BaseExecutionTimePredictorConfig, @@ -11,6 +12,8 @@ ) from vidur.entities import Batch, ExecutionTime +logger = init_logger(__name__) + # 返回单个micro-batch在单个TP shard，单个PP stage上的执行时间 # Returns execution time for a single micro-batch on a single TP shard and a single PP stage @@ -61,60 +64,103 @@ def get_execution_time(self, batch: Batch, pipeline_stage: int) -> ExecutionTime if self._config.backend == "simai_simulation": tensor_parallel_communication_time = self._tp_time_predictor.get_execution_time(batch) - # TODO: chentong fix it - # fy：有可能跑出来结果是-1 - # fy: Result may be -1 - assert tensor_parallel_communication_time >= 0, "> Debug: tensor_parallel_communication_time must be greater than 0" + # TODO(chentong): fix potential -1 return value + # Result may be -1 in some cases + # 有可能跑出来结果是 -1 + assert tensor_parallel_communication_time >= 0, "tensor_parallel_communication_time must be non-negative" - # >: 如果simai 后端返回-1，则调用vidur的查表方法 - # >: If simai backend returns -1, call vidur's lookup table method + # If simai backend returns -1, fall back to vidur's lookup table method + # 如果 simai 后端返回 -1，则调用 vidur 的查表方法 if tensor_parallel_communication_time == -1: tensor_parallel_communication_time = self._get_tensor_parallel_communication_time(batch) # elif self._config.simai_analytical_enable: elif self._config.backend == "simai_analytical": tensor_parallel_communication_time = self._tp_time_predictor.get_execution_time_by_simai_analytical(batch) - assert tensor_parallel_communication_time >= 0, "> Debug: tensor_parallel_communication_time must be greater than 0" + assert tensor_parallel_communication_time >= 0, "tensor_parallel_communication_time must be non-negative" - # >：如果simai 后端返回-1，则调用vidur的查表方法 - # >: If simai backend returns -1, call vidur's lookup table method + # If simai backend returns -1, fall back to vidur's lookup table method + # 如果 simai 后端返回 -1，则调用 vidur 的查表方法 if tensor_parallel_communication_time == -1: tensor_parallel_communication_time = self._get_tensor_parallel_communication_time(batch) elif self._config.backend == "aicb": - # TODO currently not supported TP communication when using aicb + # TODO(tianhao909): add TP communication support for AICB backend + # TODO(tianhao909): AICB 后端暂不支持 TP 通信 tensor_parallel_communication_time = 0 else: - assert self._config.backend == "vidur", "> Debug: self._config.backend can only be simai_simulation, simai_analytical, vidur" + assert self._config.backend == "vidur", "backend must be one of: simai_simulation, simai_analytical, aicb, vidur" tensor_parallel_communication_time = self._get_tensor_parallel_communication_time(batch) if self._config.backend == "aicb": - # > add self - # extract AICB params + # ============================================================ + # [AICB Backend] Build per-batch replica_config copy + # Need to set correct params based on current batch phase (prefill/decode) + # [AICB Backend] 构建 per-batch 的 replica_config 副本 + # 需要根据当前 batch 的 phase (prefill/decode) 设置正确参数 + # ============================================================ import copy replica_config = copy.deepcopy(self._replica_config) - # TODO is this correct? + # Determine current batch phase: prefill or decode + # 判断当前 batch 的 phase: prefill or decode batch_prefill_replica_id = batch.requests[0].prefill_replica_id batch_replica_id = batch.replica_id - # > add - # print(f"> debug self.replica_type ") - - if batch_prefill_replica_id == batch_replica_id: replica_config.phase = "prefill" else: replica_config.phase = "decode" - - tp = self._replica_config.tensor_parallel_size - pp = self._replica_config.num_pipeline_stages - # dp = 1 # TODO get world_size from dp or somehow get replica size - dp = self.simulation_config.cluster_config.num_replicas - ws = tp * pp * dp + + # ============================================================ + # [PD-Aware] Set correct TP/PP/WS/EP per phase + # PD separation: prefill/decode have independent world_size and EP + # - prefill: ws = p_tp * p_pp * num_p, ep = ws + # - decode: ws = d_tp * d_pp * num_d, ep = ws + # Non-PD: ws = tp * pp * total_dp, ep = ws + # + # [PD-Aware] 按 phase 设置正确的 TP/PP/WS/EP + # PD 分离时: prefill/decode 有独立的 world_size 和 EP + # 非 PD 场景: ws = tp * pp * total_dp, ep = ws + # ============================================================ + orig_tp = self._replica_config.tensor_parallel_size + orig_pp = self._replica_config.num_pipeline_stages + total_dp = self.simulation_config.cluster_config.num_replicas + + if replica_config.phase == "prefill" and hasattr(self._replica_config, 'prefill_world_size'): + # PD separation: use prefill cluster params / PD 分离: 使用 prefill 集群的参数 + tp = getattr(self._replica_config, '_prefill_tp', orig_tp) + pp = getattr(self._replica_config, '_prefill_pp', orig_pp) + ws = self._replica_config.prefill_world_size + ep = getattr(self._replica_config, 'prefill_ep', ws) + elif replica_config.phase == "decode" and hasattr(self._replica_config, 'decode_world_size'): + # PD separation: use decode cluster params / PD 分离: 使用 decode 集群的参数 + tp = getattr(self._replica_config, '_decode_tp', orig_tp) + pp = getattr(self._replica_config, '_decode_pp', orig_pp) + ws = self._replica_config.decode_world_size + ep = getattr(self._replica_config, 'decode_ep', ws) + else: + # Non-PD: EP = ws = tp * pp * dp / 非 PD 场景 + tp = orig_tp + pp = orig_pp + ws = tp * pp * total_dp + ep = ws + + # Write per-phase params to copied replica_config + # 将 per-phase 参数写入 copy 后的 replica_config replica_config.world_size = ws - + replica_config.expert_model_parallel_size = ep + replica_config.tensor_parallel_size = tp + replica_config.num_pipeline_stages = pp + + # Print current batch AICB params for debugging + # Note: non-PD mode also has prefill_world_size (unified interface), use pd_node_ratio to determine + # 打印当前 batch 的 AICB 参数, 方便调试确认 + # 注意: 非PD模式也有 prefill_world_size (统一接口), 用 pd_node_ratio 判断 + pd_mode = "PD-separated" if self._replica_config.pd_node_ratio < 1 else "MIXED(non-PD)" + logger.debug(f"[AICB Params] phase={replica_config.phase}, tp={tp}, pp={pp}, " + f"ws={ws}, ep={ep}, total_dp={total_dp}, mode={pd_mode}") if replica_config.phase == "prefill": bs = 1 seq = 0 @@ -122,13 +168,29 @@ def get_execution_time(self, batch: Batch, pipeline_stage: int) -> ExecutionTime if request._is_prefill_complete: continue seq += num_tokens_to_process + # Prefill phase does not need first-last interpolation + # prefill阶段不需要首尾插值 + replica_config.decode_last_seq = None elif replica_config.phase == "decode": bs = 0 seq = 0 + decode_last_seq = 0 # Last decode iteration's seq for first-last interpolation / 最后一轮decode的seq值，用于首尾插值预加载 for request, num_tokens_to_process in zip(batch.requests, batch.num_tokens): if request._is_prefill_complete: bs += 1 + # Current iteration seq = prefill_tokens + processed_decode_tokens - 1 + # 当前迭代的seq = prefill_tokens + processed_decode_tokens - 1 seq += request.num_processed_prefill_tokens + request.num_processed_decode_tokens - 1 + # Last iteration seq = prefill_tokens + (decode_tokens - 1) - 1 + # Because at last iteration processed_decode_tokens = decode_tokens - 1 + # 最后一轮的seq = prefill_tokens + (decode_tokens - 1) - 1 + # 因为最后一轮时 processed_decode_tokens = decode_tokens - 1 + decode_last_seq += request.num_processed_prefill_tokens + (request.num_decode_tokens - 1) - 1 + + # [First-Last Interpolation] Save last decode iteration seq + # [首尾插值] 保存最后一轮decode的seq值 + replica_config.decode_last_seq = decode_last_seq + logger.debug(f"[AICB first-last interpolation] decode current seq={seq}, last iter seq={decode_last_seq}") replica_config.batch_size = bs replica_config.seq_len = seq @@ -160,6 +222,7 @@ def get_execution_time(self, batch: Batch, pipeline_stage: int) -> ExecutionTime self.replica_scheduler_config # self._model_config ) + else: return ExecutionTime( self._num_layers_per_pipeline_stage, diff --git a/vidur-alibabacloud/vidur/execution_time_predictor/communication_time_predictor.py b/vidur-alibabacloud/vidur/execution_time_predictor/communication_time_predictor.py index 4da6d3ad..160148bd 100644 --- a/vidur-alibabacloud/vidur/execution_time_predictor/communication_time_predictor.py +++ b/vidur-alibabacloud/vidur/execution_time_predictor/communication_time_predictor.py @@ -25,8 +25,8 @@ def __init__(self, self.predictor_config = predictor_config self.replica_config = replica_config # TODO ct: change to sizeof(tensor.dtype) - # fy：得做； 动态调整dtype； 从config里面获取 - # fy: need to do; dynamically adjust dtype; get from config + # TODO: dynamically adjust dtype from config + # 待实现: 从 config 动态获取 dtype self.tensor_size = 2 self.workload: SimAIWorkload = SimAIWorkload( tp_size=replica_config.tensor_parallel_size, @@ -79,8 +79,8 @@ def get_execution_time(self, batch: Batch): # Generate hash based on all relevant parameters of WorkItem - # TODO: > 增加layer0 str（1） "ALLREDUCE"等其他变量对于hash的影响，目前是写成固定值的 - # TODO: > Add impact of other variables like layer0 str(1) "ALLREDUCE" to hash, currently hardcoded + # TODO(tianhao909): add layer0 str(1) "ALLREDUCE" etc. to hash computation, currently hardcoded + # TODO(tianhao909): 增加 layer0 str(1) "ALLREDUCE" 等变量对 hash 的影响，目前是固定值 work_item_data = ( "layer0" + @@ -173,13 +173,13 @@ def get_execution_time(self, batch: Batch): # > rewrite: add two features to reuse same workloads and results of same commands def get_execution_time_by_simai_analytical(self, batch: Batch): """ + Predict communication time using SimAI analytical tool. + Args: batch: Batch object containing batch information to process. + Returns: float: Predicted execution time (ms), returns -1 on error. + 使用SimAI分析工具预测通信时间的方法 Args: batch: Batch对象，包含需要处理的批次信息 Returns: float: 预测的执行时间（毫秒），如果出错则返回-1 - - The method of predicting communication time using the SimAI analysis tool - Args: batch: Batch object, containing the batch information to be processed. - Returns: float: Predicted execution time (milliseconds), returns -1 if an error occurs. """ self.workload.flush() num_tokens_in_batch = batch._total_num_tokens_rounded @@ -198,12 +198,12 @@ def get_execution_time_by_simai_analytical(self, batch: Batch): * self.replica_config.tensor_parallel_size**1.25) - # 为workload和命令生成唯一标识符 - # 基于WorkItem的所有相关参数生成哈希值 - # TODO: > 增加layer0 str（1） "ALLREDUCE"等其他变量对于hash的影响，目前是写成固定值的 + # TODO(tianhao909): add layer0 str(1) "ALLREDUCE" etc. to hash computation, currently hardcoded + # 为 workload 和命令生成唯一标识符 # Generate unique identifier for workload and command + # 基于 WorkItem 的所有相关参数生成哈希值 # Generate hash based on all relevant parameters of WorkItem - # TODO: > Add impact of other variables like layer0 str(1) "ALLREDUCE" to hash, currently hardcoded + # TODO(tianhao909): 增加 layer0 str(1) "ALLREDUCE" 等变量对 hash 的影响，目前是固定值 work_item_data = ( "layer0" + @@ -214,6 +214,7 @@ def get_execution_time_by_simai_analytical(self, batch: Batch): str(self.tensor_size) ) + # Generate MD5 hash as workload identifier # 使用MD5哈希算法生成工作负载标识符 workload_identifier = hashlib.md5(work_item_data.encode()).hexdigest() @@ -301,9 +302,8 @@ def get_execution_time_by_simai_analytical(self, batch: Batch): # Get latency data from index 5 position (microseconds), convert to milliseconds latency = float(rows[-1][5]) * 1e-3 - # TODO: > 量太小有可能tp通信量是零， 比如 通信量为65535的时候， 就是0 - # 通信量为3276800的时候， laytency=0.015ms - # TODO: > Amount may be too small for tp communication to be zero, e.g. when communication amount is 65535, it's 0 + # TODO(tianhao909): handle near-zero TP communication amounts (e.g. 65535 bytes -> 0 latency) + # TODO(tianhao909): 通信量太小时 TP 通信可能为 0，如 65535 bytes 时 latency=0 # When communication amount is 3276800, latency=0.015ms # assert all_reduce_bytes>0 and latency > 0, f"> Debug: all_reduce_bytes={all_reduce_bytes} latency={latency} need to be >=0" diff --git a/vidur-alibabacloud/vidur/execution_time_predictor/sklearn_execution_time_predictor.py b/vidur-alibabacloud/vidur/execution_time_predictor/sklearn_execution_time_predictor.py index b46a4f03..a3e505af 100644 --- a/vidur-alibabacloud/vidur/execution_time_predictor/sklearn_execution_time_predictor.py +++ b/vidur-alibabacloud/vidur/execution_time_predictor/sklearn_execution_time_predictor.py @@ -681,9 +681,10 @@ def _train_models(self) -> Dict[str, BaseEstimator]: return models def _predict_for_compute_models_by_aicb(self) -> Dict[str, Any]: - # 存储预测结果 + # Store prediction results / 存储预测结果 predictions = {} + # Define compute layer model names for prediction # 定义需要预测的计算层模型名 model_names = [ "attn_pre_proj", @@ -691,84 +692,100 @@ def _predict_for_compute_models_by_aicb(self) -> Dict[str, Any]: "mlp_up_proj", "mlp_down_proj", "mlp_act", - # "attn_rope", # 当前未启用RoPE层建模 + # "attn_rope", # RoPE layer modeling not enabled / 当前未启用RoPE层建模 "attn_kv_cache_save", "input_layernorm", "post_attention_layernorm", "add", ] + # Add send/recv comm model if pipeline parallelism exists # 若存在流水线并行，则加入send/recv通信模型 if self._replica_config.num_pipeline_stages > 1: model_names.append("send_recv") + # Add all_reduce comm model if tensor parallelism exists # 若存在张量并行，则加入all_reduce通信模型 if self._replica_config.tensor_parallel_size > 1: model_names.append("all_reduce") + # Generate range from 1 to max tokens for batch prediction # 生成从1到最大token数的范围，用于批量预测 num_token_range = np.arange(1, self._max_tokens + 1) - # 构造输入DataFrame + # Construct input DataFrame / 构造输入DataFrame X = pd.DataFrame({"num_tokens": num_token_range}) + # Predict and cache results for each compute model # 对每个计算模型进行预测，并将结果缓存 for model_name in model_names: + # Create simulated predictions with linear growth based on num_tokens + # Set different base time and growth rate per model type # 创建模拟的预测值，基于num_tokens线性增长 # 根据模型类型设置不同的基础时间和增长率 if model_name in ["attn_pre_proj", "attn_post_proj"]: + # Attention projection: base 0.001ms, +0.0001ms per token # 注意力投影层：基础时间0.001ms，每token增加0.0001ms predictions[model_name] = { (num_tokens,): 0.001 + 0.0001 * num_tokens for num_tokens in num_token_range } elif model_name in ["mlp_up_proj", "mlp_down_proj"]: + # MLP layer: base 0.0015ms, +0.00015ms per token # MLP层：基础时间0.0015ms，每token增加0.00015ms predictions[model_name] = { (num_tokens,): 0.0015 + 0.00015 * num_tokens for num_tokens in num_token_range } elif model_name == "mlp_act": + # MLP activation: base 0.0005ms, +0.00005ms per token # MLP激活层：基础时间0.0005ms，每token增加0.00005ms predictions[model_name] = { (num_tokens,): 0.0005 + 0.00005 * num_tokens for num_tokens in num_token_range } elif model_name == "attn_kv_cache_save": + # KV cache save: base 0.0002ms, +0.00002ms per token # KV缓存保存：基础时间0.0002ms，每token增加0.00002ms predictions[model_name] = { (num_tokens,): 0.0002 + 0.00002 * num_tokens for num_tokens in num_token_range } elif model_name in ["input_layernorm", "post_attention_layernorm"]: + # LayerNorm: base 0.0003ms, +0.00003ms per token # LayerNorm层：基础时间0.0003ms，每token增加0.00003ms predictions[model_name] = { (num_tokens,): 0.0003 + 0.00003 * num_tokens for num_tokens in num_token_range } elif model_name == "add": + # Add op: base 0.0001ms, +0.00001ms per token # Add操作：基础时间0.0001ms，每token增加0.00001ms predictions[model_name] = { (num_tokens,): 0.0001 + 0.00001 * num_tokens for num_tokens in num_token_range } elif model_name == "send_recv": + # Send/Recv comm: base 0.01ms, +0.001ms per token # Send/Recv通信：基础时间0.01ms，每token增加0.001ms predictions[model_name] = { (num_tokens,): 0.01 + 0.001 * num_tokens for num_tokens in num_token_range } elif model_name == "all_reduce": + # All-reduce comm: base 0.02ms, +0.002ms per token # All-reduce通信：基础时间0.02ms，每token增加0.002ms predictions[model_name] = { (num_tokens,): 0.02 + 0.002 * num_tokens for num_tokens in num_token_range } + # Predict and cache results for each compute model # 对每个计算模型进行预测，并将结果缓存 # for model_name in model_names: # model = self._models[model_name] # predictions[model_name] = self._get_model_prediction(model_name, model, X) + # Return all compute layer predictions (for later lookup) # 返回所有计算相关层的预测结果（用于后续查表） return predictions @@ -809,14 +826,14 @@ def _predict_for_compute_models(self) -> Dict[str, Any]: def _predict_for_cpu_overhead_models_by_aicb(self) -> Dict[str, Any]: - # 若跳过CPU开销建模，则直接返回空 + # Skip if CPU overhead modeling is disabled / 若跳过CPU开销建模，则直接返回空 if self._config.skip_cpu_overhead_modeling: return {} - # 存储CPU开销预测结果 + # Store CPU overhead predictions / 存储CPU开销预测结果 predictions = {} - # CPU相关的开销模型名称 + # CPU-related overhead model names / CPU相关的开销模型名称 model_names = [ "schedule", "sampler_e2e", @@ -825,52 +842,58 @@ def _predict_for_cpu_overhead_models_by_aicb(self) -> Dict[str, Any]: "ray_comm_time", ] + # Batch size range: 1 to max prediction batch size # 批处理大小范围：1 到 最大预测批大小 batch_size_range = np.arange(1, self._config.prediction_max_batch_size + 1) - # 构造输入数据 + # Construct input data / 构造输入数据 X = pd.DataFrame({"batch_size": batch_size_range}) - # 对每个CPU开销模型进行预测 + # Predict for each CPU overhead model / 对每个CPU开销模型进行预测 # for model_name in model_names: # model = self._models[model_name] # predictions[model_name] = self._get_model_prediction(model_name, model, X) - # 对每个CPU开销模型进行预测 + # Predict for each CPU overhead model / 对每个CPU开销模型进行预测 for model_name in model_names: - # 为每个模型生成模拟值 + # Generate simulated values for each model / 为每个模型生成模拟值 if model_name == "schedule": + # Scheduling overhead: base 0.5ms, +0.05ms per batch # 调度开销：基础时间0.5ms，每增加一个batch增加0.05ms predictions[model_name] = { (batch_size,): 0.5 + 0.05 * batch_size for batch_size in batch_size_range } elif model_name == "sampler_e2e": + # Sampler e2e overhead: base 1.0ms, +0.1ms per batch # 采样端到端开销：基础时间1.0ms，每增加一个batch增加0.1ms predictions[model_name] = { (batch_size,): 1.0 + 0.1 * batch_size for batch_size in batch_size_range } elif model_name == "prepare_inputs_e2e": + # Prepare inputs e2e overhead: base 0.8ms, +0.08ms per batch # 准备输入端到端开销：基础时间0.8ms，每增加一个batch增加0.08ms predictions[model_name] = { (batch_size,): 0.8 + 0.08 * batch_size for batch_size in batch_size_range } elif model_name == "process_model_outputs": + # Process model outputs overhead: base 0.3ms, +0.03ms per batch # 处理模型输出开销：基础时间0.3ms，每增加一个batch增加0.03ms predictions[model_name] = { (batch_size,): 0.3 + 0.03 * batch_size for batch_size in batch_size_range } elif model_name == "ray_comm_time": + # Ray comm time: base 0.2ms, +0.02ms per batch # Ray通信时间：基础时间0.2ms，每增加一个batch增加0.02ms predictions[model_name] = { (batch_size,): 0.2 + 0.02 * batch_size for batch_size in batch_size_range } - # 返回CPU开销部分的预测结果 + # Return CPU overhead predictions / 返回CPU开销部分的预测结果 return predictions def _predict_for_cpu_overhead_models(self) -> Dict[str, Any]: @@ -897,21 +920,23 @@ def _predict_for_cpu_overhead_models(self) -> Dict[str, Any]: return predictions def _predict_for_attention_layer_models_by_aicb(self) -> Dict[str, Any]: - # 存储注意力层预测结果 + # Store attention layer predictions / 存储注意力层预测结果 predictions = {} - # 解码阶段的batch size枚举范围 + # Decode batch size enumeration range / 解码阶段的batch size枚举范围 decode_batch_size_range = np.arange( 1, self._config.prediction_max_batch_size + 1 ) + # KV cache size range, incremented by block granularity # kv缓存大小的枚举范围，按block粒度递增 decode_kv_cache_size_range = np.arange( 0, self._config.prediction_max_tokens_per_request + 1, - self._config.kv_cache_prediction_granularity, # kv cache存储为block，这里就是block_size + self._config.kv_cache_prediction_granularity, # KV cache stored in blocks / kv cache存储为block，这里就是block_size ) - # 解码时prefill chunk size固定为0 + # Decode prefill_chunk_size is fixed at 0 / 解码时prefill chunk size固定为0 decode_prefill_chunk_size_range = [0] + # Generate all combinations (Cartesian product) # 生成所有组合（笛卡尔积） decode_batch_size, decode_kv_cache_size, decode_prefill_chunk_size = zip( *product( @@ -921,18 +946,21 @@ def _predict_for_attention_layer_models_by_aicb(self) -> Dict[str, Any]: ) ) + # Prefill only supports batch_size=1 (single request typically) # Prefill阶段仅支持batch_size=1（通常单请求） prefill_batch_size_range = [1] + # Also enumerate KV cache size by block granularity # 同样按block粒度枚举kv缓存大小 prefill_kv_cache_size_range = np.arange( 0, self._config.prediction_max_tokens_per_request + 1, self._config.kv_cache_prediction_granularity, ) - # Prefill chunk大小从1到最大允许值 + # Prefill chunk size from 1 to max / Prefill chunk大小从1到最大允许值 prefill_prefill_chunk_size_range = np.arange( 1, self._config.prediction_max_prefill_chunk_size + 1 ) + # Generate all prefill parameter combinations # 生成所有prefill参数组合 prefill_batch_size, prefill_kv_cache_size, prefill_prefill_chunk_size = zip( *product( @@ -942,6 +970,7 @@ def _predict_for_attention_layer_models_by_aicb(self) -> Dict[str, Any]: ) ) + # Merge decode and prefill combinations into one DataFrame # 合并decode和prefill的所有参数组合成一个DataFrame attention_df = pd.DataFrame( { @@ -952,47 +981,56 @@ def _predict_for_attention_layer_models_by_aicb(self) -> Dict[str, Any]: } ) - # 标记是否为decode阶段（chunk_size == 0） + # Mark decode stage (chunk_size == 0) / 标记是否为decode阶段 attention_df["is_decode"] = attention_df["prefill_chunk_size"] == 0 + # num_tokens = max(prefill_chunk_size, batch_size) # num_tokens取prefill_chunk_size和batch_size的最大值 attention_df["num_tokens"] = attention_df[ ["prefill_chunk_size", "batch_size"] ].max(axis=1) + # Add squared prefill chunk size feature (for model input) # 添加prefill chunk大小的平方项（用于模型输入） attention_df["prefill_chunk_size_squared"] = ( attention_df["prefill_chunk_size"] ** 2 ) - # 分离出prefill和decode的数据子集 + # Split into prefill and decode subsets / 分离出prefill和decode的数据子集 prefill_df = attention_df[~attention_df["is_decode"]] decode_df = attention_df[attention_df["is_decode"]] - # 进一步筛选有缓存的prefill数据 + # Filter prefill data with existing cache / 进一步筛选有缓存的prefill数据 chunked_prefill_df = prefill_df[prefill_df["kv_cache_size"] > 0].copy() - # 计算总的prefill token数量 + # Calculate total prefill token count / 计算总的prefill token数量 chunked_prefill_df["total_prefill_tokens"] = ( chunked_prefill_df["kv_cache_size"] + chunked_prefill_df["prefill_chunk_size"] ) + # Predict prefill attention time using simulated values (based on kv_cache and chunk^2) # 使用模拟值预测prefill注意力时间（基于kv缓存和chunk^2） prefill_data = prefill_df[["kv_cache_size", "prefill_chunk_size_squared"]].values predictions["attn_prefill"] = {} for kv_cache_size, prefill_chunk_size_squared in prefill_data: + # Prefill time based on kv_cache size and chunk size squared + # base 0.01ms + kv_cache/100 * 0.001ms + chunk_size_sq/10000 * 0.0001ms # Prefill时间基于kv_cache大小和chunk大小平方计算 # 基础时间0.01ms + kv_cache每增加100增加0.001ms + chunk_size_squared每增加10000增加0.0001ms time = 0.01 + (kv_cache_size * 0.001 / 100) + (prefill_chunk_size_squared * 0.0001 / 10000) predictions["attn_prefill"][(int(kv_cache_size), int(prefill_chunk_size_squared))] = time + # Predict decode attention time using simulated values (based on batch_size and kv_cache) # 使用模拟值预测decode注意力时间（基于batch_size和kv缓存） decode_data = decode_df[["batch_size", "kv_cache_size"]].values predictions["attn_decode"] = {} for batch_size, kv_cache_size in decode_data: + # Decode time based on batch_size and kv_cache size + # base 0.005ms + 0.002ms per batch_size + kv_cache/100 * 0.0005ms # Decode时间基于batch_size和kv_cache大小计算 # 基础时间0.005ms + batch_size每增加1增加0.002ms + kv_cache_size每增加100增加0.0005ms time = 0.005 + (batch_size * 0.002) + (kv_cache_size * 0.0005 / 100) predictions["attn_decode"][(int(batch_size), int(kv_cache_size))] = time + # # Predict prefill attention time using trained model (based on kv_cache and chunk^2) # # 使用训练好的模型预测prefill注意力时间（基于kv缓存和chunk^2） # predictions["attn_prefill"] = self._get_model_prediction( # "attn_prefill", @@ -1000,6 +1038,7 @@ def _predict_for_attention_layer_models_by_aicb(self) -> Dict[str, Any]: # prefill_df[["kv_cache_size", "prefill_chunk_size_squared"]], # ) + # # Predict decode attention time using trained model (based on batch_size and kv_cache) # # 使用训练好的模型预测decode注意力时间（基于batch_size和kv缓存） # predictions["attn_decode"] = self._get_model_prediction( # "attn_decode", @@ -1007,7 +1046,7 @@ def _predict_for_attention_layer_models_by_aicb(self) -> Dict[str, Any]: # decode_df[["batch_size", "kv_cache_size"]], # ) - # 返回注意力层所有预测结果 + # Return all attention layer predictions / 返回注意力层所有预测结果 return predictions def _predict_for_attention_layer_models(self) -> Dict[str, Any]: @@ -1235,7 +1274,8 @@ def _get_attention_decode_execution_time(self, batch: Batch) -> float: ) = self._get_batch_decode_attention_params(batch) if decode_batch_size == 0: return 0 - #TODO decode 打印 + # TODO(tianhao909): add decode output logging + # TODO(tianhao909): 添加 decode 输出日志 return self._predictions["attn_decode"][ (decode_batch_size, decode_avg_kv_cache_size) ] * ( diff --git a/vidur-alibabacloud/vidur/metrics/cdf_sketch.py b/vidur-alibabacloud/vidur/metrics/cdf_sketch.py index 50aeebf2..327de59c 100644 --- a/vidur-alibabacloud/vidur/metrics/cdf_sketch.py +++ b/vidur-alibabacloud/vidur/metrics/cdf_sketch.py @@ -5,6 +5,7 @@ from ddsketch.ddsketch import DDSketch from vidur.logger import init_logger +from vidur.metrics.data_series import _safe_write_image # qoder logger = init_logger(__name__) @@ -146,5 +147,6 @@ def plot_cdf(self, path: str, plot_name: str, x_axis_label: str = None) -> None: labels={"x": x_axis_label}, ) fig.update_traces(marker=dict(color="red", size=2)) - fig.write_image(f"{path}/{plot_name}.png") + # fig.write_image(f"{path}/{plot_name}.png") + _safe_write_image(fig, f"{path}/{plot_name}.png") # qoder self._save_df(df, path, plot_name) diff --git a/vidur-alibabacloud/vidur/metrics/data_series.py b/vidur-alibabacloud/vidur/metrics/data_series.py index 51be848d..3e5648e2 100644 --- a/vidur-alibabacloud/vidur/metrics/data_series.py +++ b/vidur-alibabacloud/vidur/metrics/data_series.py @@ -11,6 +11,34 @@ logger = init_logger(__name__) +# Chrome/Kaleido 是否可用的标记, 首次失败后跳过后续所有 write_image 调用 +# Flag for Chrome/Kaleido availability, skip all subsequent write_image after first failure +_KALEIDO_AVAILABLE = True + + +def _safe_write_image(fig, path: str): + """ + 安全地写入图片, Chrome/Kaleido 不可用时优雅跳过 + Safely write image, gracefully skip when Chrome/Kaleido is unavailable + """ + global _KALEIDO_AVAILABLE + if not _KALEIDO_AVAILABLE: + return + try: + fig.write_image(path) + except RuntimeError as e: + if "Chrome" in str(e) or "Kaleido" in str(e): + _KALEIDO_AVAILABLE = False + logger.warning( + f"[Plot] Chrome/Kaleido 不可用, 跳过 PNG 生成. " + f"运行 'plotly_get_chrome' 安装 Chrome 后可恢复. " + f"CSV 数据仍会正常保存." + ) + else: + raise + + + class DataSeries: def __init__( self, @@ -83,6 +111,11 @@ def print_series_stats( if y_name is None: y_name = self._y_name + + # 跳过非数值列的统计 | Skip statistics for non-numeric columns + if not pd.api.types.is_numeric_dtype(df[y_name]): + logger.debug(f"{plot_name}: {y_name} is non-numeric, skipping stats") + return logger.debug( f"{plot_name}: {y_name} stats:" @@ -108,6 +141,12 @@ def print_distribution_stats( if y_name is None: y_name = self._y_name + + # 跳过非数值列的统计 (如 pd_p2p_comm_dtype='fp8' 等字符串指标) + # Skip statistics for non-numeric columns (e.g., string metrics) + if not pd.api.types.is_numeric_dtype(df[y_name]): + logger.debug(f"{plot_name}: {y_name} is non-numeric, skipping stats") + return logger.debug( f"{plot_name}: {y_name} stats:" @@ -207,7 +246,8 @@ def plot_step( labels={"x": y_axis_label}, ) fig.update_traces(marker=dict(color="red", size=2)) - fig.write_image(f"{path}/{plot_name}.png") + # fig.write_image(f"{path}/{plot_name}.png") + _safe_write_image(fig, f"{path}/{plot_name}.png") self._save_df(df, path, plot_name) @@ -219,6 +259,11 @@ def plot_cdf(self, path: str, plot_name: str, y_axis_label: str = None) -> None: y_axis_label = self._y_name df = self._to_df() + + # 跳过非数值列 | Skip non-numeric columns + if not pd.api.types.is_numeric_dtype(df[self._y_name]): + self._save_df(df, path, plot_name) + return self.print_distribution_stats(df, plot_name) @@ -252,7 +297,8 @@ def plot_cdf(self, path: str, plot_name: str, y_axis_label: str = None) -> None: df, x=self._y_name, y="cdf", markers=True, labels={"x": y_axis_label} ) fig.update_traces(marker=dict(color="red", size=2)) - fig.write_image(f"{path}/{plot_name}.png") + # fig.write_image(f"{path}/{plot_name}.png") + _safe_write_image(fig, f"{path}/{plot_name}.png") self._save_df(df, path, plot_name) def plot_histogram(self, path: str, plot_name: str) -> None: @@ -260,6 +306,10 @@ def plot_histogram(self, path: str, plot_name: str) -> None: return df = self._to_df() + + # 跳过非数值列 | Skip non-numeric columns + if not pd.api.types.is_numeric_dtype(df[self._y_name]): + return self.print_distribution_stats(df, plot_name) @@ -292,7 +342,8 @@ def plot_histogram(self, path: str, plot_name: str) -> None: if self._save_plots: fig = px.histogram(df, x=self._y_name, nbins=25) - fig.write_image(f"{path}/{plot_name}.png") + # fig.write_image(f"{path}/{plot_name}.png") + _safe_write_image(fig, f"{path}/{plot_name}.png") def plot_differential(self, path: str, plot_name: str) -> None: if len(self._data_series) == 0: @@ -333,6 +384,7 @@ def plot_differential(self, path: str, plot_name: str) -> None: if self._save_plots: fig = px.line(df, x=self._x_name, y=differential_col_name, markers=True) fig.update_traces(marker=dict(color="red", size=2)) - fig.write_image(f"{path}/{plot_name}.png") + # fig.write_image(f"{path}/{plot_name}.png") + _safe_write_image(fig, f"{path}/{plot_name}.png") self._save_df(df, path, plot_name) diff --git a/vidur-alibabacloud/vidur/metrics/metrics_store.py b/vidur-alibabacloud/vidur/metrics/metrics_store.py index 88db5c82..601d038e 100644 --- a/vidur-alibabacloud/vidur/metrics/metrics_store.py +++ b/vidur-alibabacloud/vidur/metrics/metrics_store.py @@ -10,6 +10,7 @@ from vidur.entities import Batch, BatchStage, ExecutionTime, Request from vidur.logger import init_logger from vidur.metrics.cdf_sketch import CDFSketch +from vidur.metrics.data_series import _safe_write_image # qoder from vidur.metrics.constants import ( BatchMetricsCountDistribution, BatchMetricsTimeDistribution, @@ -300,6 +301,7 @@ def _store_bar_plot( labels={"x": x_label, "y": y_label}, ) fig.write_image(f"{base_path}/{plot_name}.png") + _safe_write_image(fig, f"{base_path}/{plot_name}.png") # qoder def _store_operation_metrics(self, base_plot_path: str): if not self._config.store_operation_metrics: @@ -369,7 +371,6 @@ def _store_operation_metrics(self, base_plot_path: str): def _store_request_metrics(self, base_plot_path: str): if not self._config.store_request_metrics: return - # import pdb; pdb.set_trace() # > debug all_request_metrics = list( self._request_metrics_time_distributions.values() ) + list(self._request_metrics_histogram.values()) diff --git a/vidur-alibabacloud/vidur/profiling/collectives/benchmark_runner.py b/vidur-alibabacloud/vidur/profiling/collectives/benchmark_runner.py index 05e921a4..354f3a04 100644 --- a/vidur-alibabacloud/vidur/profiling/collectives/benchmark_runner.py +++ b/vidur-alibabacloud/vidur/profiling/collectives/benchmark_runner.py @@ -86,8 +86,8 @@ def _init_communication( f"ip_addr: {ray.util.get_node_ip_address()}, CUDA_VISIBLE_DEVICES: {os.environ['CUDA_VISIBLE_DEVICES']}" ) - # TODO > 可以改成deepep后端 - # TODO > can be changed to deepep backend + # TODO(tianhao909): support DeepEP backend + # TODO(tianhao909): 支持 DeepEP 后端 torch.distributed.init_process_group( backend="nccl", rank=rank, diff --git a/vidur-alibabacloud/vidur/profiling/collectives/collectives_impl.py b/vidur-alibabacloud/vidur/profiling/collectives/collectives_impl.py index a159f602..477188bf 100644 --- a/vidur-alibabacloud/vidur/profiling/collectives/collectives_impl.py +++ b/vidur-alibabacloud/vidur/profiling/collectives/collectives_impl.py @@ -37,9 +37,8 @@ def __init__( dtype=dtype, device="cuda", ) - # TODO > elif collective == "all_to_all": + # TODO(tianhao909): add all_to_all collective support # elif collective == "all_to_all": - # # TODO > change _reduce_buffer to what? all to all buffer? # self._reduce_buffer = torch.empty( # size=(size * num_workers,), # dtype=dtype, @@ -61,7 +60,6 @@ def __init__( if not self._disable_graph: self._graph = self._build_graph() - # > self._num_workers = num_workers def _run_all_reduce(self): @@ -80,10 +78,11 @@ def _run_send_recv(self): torch.distributed.recv(self._buffer, 0) def _run_reduce_scatter(self): - # > torch.distributed function: def reduce_scatter_tensor(output, input, op=ReduceOp.SUM, group=None, async_op=False): + # torch.distributed function: reduce_scatter_tensor torch.distributed.reduce_scatter_tensor(self._buffer, self._reduce_buffer) - # TODO > modify according to def all_to_all(output_tensor_list, input_tensor_list, group=None, async_op=False): + # TODO(tianhao909): implement all_to_all collective + # TODO(tianhao909): 实现 all_to_all 集合通信 # Or use all_to_all_single first? # def all_to_all_single( # output, diff --git a/vidur-alibabacloud/vidur/scheduler/global_scheduler/splitwise_global_scheduler.py b/vidur-alibabacloud/vidur/scheduler/global_scheduler/splitwise_global_scheduler.py index dd00de19..12092564 100644 --- a/vidur-alibabacloud/vidur/scheduler/global_scheduler/splitwise_global_scheduler.py +++ b/vidur-alibabacloud/vidur/scheduler/global_scheduler/splitwise_global_scheduler.py @@ -1,6 +1,6 @@ from typing import List, Tuple, Dict -# from vidur.config import Config +from vidur.logger import init_logger from vidur.entities import Replica, Request from vidur.scheduler.global_scheduler.base_global_scheduler import BaseGlobalScheduler @@ -14,6 +14,8 @@ ReplicaSchedulerRegistry, ) +logger = init_logger(__name__) + # > from vidur.entities.task import Task, TaskType from vidur.entities.flow import Flow, FlowType @@ -21,8 +23,8 @@ from vidur.entities.replica import Replica, ReplicaType from vidur.entities.request import Request, RequestType -# TODO: > > 参考 sw写的；但也很多区别； 换一个名字； 类似pd分离的其他名字； 不严格是sw了 -# TODO: > > Refer to sw implementation; but there are many differences; need a new name; similar to pd separation; not strictly sw anymore +# TODO(tianhao909): rename class - not strictly Splitwise anymore, more like PD-separation scheduler +# TODO(tianhao909): 重命名类，已不严格是 Splitwise，更像 PD 分离调度器 class SplitwiseGlobalScheduler(BaseGlobalScheduler): # Splitwise Global Scheduler. def __init__(self, config: SimulationConfig, replicas: Dict[int, Replica]): # Call parent class initialization method @@ -33,12 +35,14 @@ def __init__(self, config: SimulationConfig, replicas: Dict[int, Replica]): self._replicas = replicas # Save replica dictionary as instance private attribute, key is replica ID, value is replica object self._num_replicas = len(self._replicas) # Calculate and save total number of replicas - # TODO > improve pd_node_ratio + # TODO(tianhao909): make pd_node_ratio configurable + # TODO(tianhao909): 优化 pd_node_ratio 的配置方式 # self.pd_node_ratio = 0.5 self.pd_node_ratio = self._replicas[0].pd_node_ratio - assert self.pd_node_ratio >= 0 and self.pd_node_ratio <= 1, "> Debug: pd_node_ratio must be between 0 and 1." + assert self.pd_node_ratio >= 0 and self.pd_node_ratio <= 1, "pd_node_ratio must be between 0 and 1" # self._sub_scheduler = self._config.splitwise_scheduler_sub_scheduler # Get sub-scheduler type from configuration - # TODO > improve _sub_scheduler flexible choice + # TODO(tianhao909): make _sub_scheduler configurable + # TODO(tianhao909): 优化 _sub_scheduler 的灵活选择 # self._sub_scheduler = "round_robin" self._sub_scheduler = "lor" @@ -148,14 +152,11 @@ def __init__(self, config: SimulationConfig, replicas: Dict[int, Replica]): # self.mixed_instances = [] # Mixed instance list (can handle prompts and tokens) # self.token_instances = [] # Token instance list - # TODO : > 增加到输入或者仿真里面 - # fy 这个需要是一个入参 从config里面读取 - # TODO : > Add to input or simulation - # fy This needs to be an input parameter read from config + # TODO(tianhao909): add transfer_bandwidth to config input + # TODO(tianhao909): 增加到输入或仿真配置中，从 config 读取 self.transfer_bandwidth = 0 self.transfer_bandwidth = 200 * 1024**3 # Gbps转换为bps - # > self.p_request_counter = 0 self.d_request_counter = 0 @@ -346,7 +347,7 @@ def schedule(self) -> List[Tuple[int, Request]]: # Execute scheduling logic met prefill_replica = None - + # fth 目前p req 进入 p replica的 lor 策略： # vidur's lor: # replica_id = min(pending_prefill_requests_map.items(), key=lambda x: x[1])[0] @@ -373,21 +374,46 @@ def schedule(self) -> List[Tuple[int, Request]]: # Execute scheduling logic met request.decode_replica_id = replica_id decode_request_mapping.append((decode_replica.id, request)) - # TODO fy： 没有用的 task dag等相关代码都可以删掉； 优先级相对低 - # task继承 req； 或者让task能构造req； - # TODO fy: Unused task dag related code can be deleted; relatively low priority + # TODO(tianhao909): remove unused task DAG code (low priority) + # TODO(tianhao909): 删除未使用的 task DAG 相关代码（优先级较低） # task inherits from req; or let task construct req + # task 继承 req；或者让 task 能构造 req + ''' if prefill_replica != decode_replica: # If prompt instance and token instance are different + # ============================================================ + # [冗余代码分析] add_to_pool + DAG + sched_* 运行时验证 + # + # 以下代码在 Splitwise PD分离 实际调度流程中并不影响核心逻辑: + # 1. add_to_pool(): 将request加入replica.pending_requests + # - prefill端: batch_end_event.py 会从中remove，但不影响调度 + # - decode端: 从未被消费，纯冗余 + # 2. request DAG (prefill_task, decode_task): 创建了DAG图， + # 但_get_next_batch()使用的是_request_queue，不读取DAG + # 3. add_kv_cache_transfer(): 构建了flow node，但模拟不使用 + # 4. sched_memory/sched_pending_tokens: 设置但从未被读取用于调度 + # + # 保留这些代码以兼容可能的上层逻辑，但标注为冗余。 + # ============================================================ + logger.debug(f"[Redundant code check (冗余代码验证)] schedule(): " + f"req={request.id}, p_replica={prefill_replica.id}, d_replica={decode_replica.id}") + logger.debug(f" add_to_pool(prefill_task): added to p_replica.pending_requests " + f"(加入 p_replica.pending_requests, len={len(prefill_replica.pending_requests)})") + logger.debug(f" add_to_pool(decode_task): added to d_replica.pending_requests " + f"(加入 d_replica.pending_requests, len={len(decode_replica.pending_requests)}) [redundant, 冗余]") prefill_replica.add_to_pool(prefill_task) - decode_replica.add_to_pool(decode_task) + # decode_replica.add_to_pool(decode_task) + decode_replica.add_to_pool(decode_task) # [冗余] decode端pending_requests从未被消费 + # [冗余] 以下KV cache transfer/DAG操作不影响实际调度 # 在实例之间传输KV缓存 # Transfer KV cache between instances self.add_kv_cache_transfer(request, prefill_replica, decode_replica, self.transfer_bandwidth) + + # [冗余] sched_memory 设置但从未被核心调度逻辑读取 prefill_replica.sched_memory += prefill_task.max_memory(prefill_replica) # Update prompt instance memory usage decode_replica.sched_memory += prefill_task.max_memory(decode_replica) + \ decode_task.max_memory(decode_replica) # Update token instance memory usage @@ -398,10 +424,11 @@ def schedule(self) -> List[Tuple[int, Request]]: # Execute scheduling logic met prefill_replica.sched_memory += prefill_task.max_memory(prefill_replica) + \ decode_task.max_memory(prefill_replica) # Update instance memory usage prefill_task.chain = [decode_task] # Set token task as successor of prompt task - + + # [冗余] sched_pending_tokens 设置但从未被核心调度逻辑读取 prefill_replica.sched_pending_tokens += prefill_task.prompt_size # Update prompt instance pending token count decode_replica.sched_pending_tokens += 1 # Update token instance pending token count - + ''' # > for req in requests_to_remove: self._request_queue.remove(req) diff --git a/vidur-alibabacloud/vidur/scheduler/replica_scheduler/splitwise_replica_scheduler.py b/vidur-alibabacloud/vidur/scheduler/replica_scheduler/splitwise_replica_scheduler.py index ff32aa57..5219f68a 100644 --- a/vidur-alibabacloud/vidur/scheduler/replica_scheduler/splitwise_replica_scheduler.py +++ b/vidur-alibabacloud/vidur/scheduler/replica_scheduler/splitwise_replica_scheduler.py @@ -19,7 +19,6 @@ from vidur.scheduler.utils.memory_planner import MemoryPlanner from vidur.scheduler.replica_scheduler.base_replica_scheduler import BaseReplicaScheduler -# > from collections import defaultdict import sys from vidur.entities.node import NodeState, Node @@ -84,8 +83,8 @@ def __init__( - # TODO: > > 没用到了 没有用的代码删一下 - # TODO: > > not used anymore - delete unused code + # TODO(tianhao909): delete unused prompt/token task tracking code + # TODO(tianhao909): 删除未使用的 prompt/token task 跟踪代码 self.prompt_tasks_in_batch = [] self.token_tasks_in_batch = [] @@ -110,15 +109,15 @@ def __init__( # 按到达时间排序的待处理请求列表 # pending requests (not tasks) ordered by arrival time - # TODO: > 看看需不需要删 - # TODO: > check if needs deletion + # TODO(tianhao909): evaluate if pending_requests can be removed + # TODO(tianhao909): 看看需不需要删除 pending_requests self.pending_requests = [] # 专门用于提示任务的待处理队列（优先处理提示） # separate pending queue for prompt tasks (to prioritize prompts) - # TODO > 删一下冗余的 - # TODO > remove redundant items + # TODO(tianhao909): remove redundant prompt queue + # TODO(tianhao909): 删除冗余的 prompt 队列 self.pending_prompt_queue = [] # 请求到任务的映射关系 @@ -199,22 +198,46 @@ def _allocate_request(self, request: Request) -> None: self.allocate(request.id, 1) # Allocate one additional memory block + # fth 260122 这边释放资源； 进入decode的逻辑 写在handle_event里面，用于生成新的event， 这边不生成event； def on_batch_end(self, batch: Batch) -> None: # Called when a batch finishes execution self._num_running_batches -= 1 # Decrement running batch count # 判断是否是pd 分离 # Check if PD separation is enabled if self.replica.replica_type == ReplicaType.MIXED: - assert False, "> debug, PD separation doesn't support mixed yet, must be separated" + assert False, "PD separation doesn't support mixed mode yet, must be separated" pass elif self.replica.replica_type == ReplicaType.PREFILL: for request in batch.requests: if request.completed: self.free(request.id) + + + # 在移除请求之前，先计算当前的kvcache使用情况 + logger.debug(f"Before removing request {request.id}:") + self.replica.get_remaining_kv_cache_capacity() + # 移除请求 留给batch_end_event.py去做 + # self.replica.pending_requests.remove(request) + # 移除请求后释放相应的显存 + self.replica.release_request_kv_cache_memory(request) + logger.debug(f"Request {request.id} removed from replica and GPU memory released") + self.replica.get_remaining_kv_cache_capacity() + elif request.is_prefill_complete == True: # 通过 request 找到对应 decode replica； # Find corresponding decode replica through request; self.free(request.id) + + # 在移除请求之前，先计算当前的kvcache使用情况 + logger.debug(f"Before removing request {request.id}:") + self.replica.get_remaining_kv_cache_capacity() + # 移除请求 留给batch_end_event.py去做 + # self.replica.pending_requests.remove(request) + # 移除请求后释放相应的显存 + self.replica.release_request_kv_cache_memory(request) + logger.debug(f"Request {request.id} removed from replica and GPU memory released") + self.replica.get_remaining_kv_cache_capacity() + d_replica_scheduler = request.global_scheduler.get_replica_scheduler(request.decode_replica_id) # d_replica_scheduler._preempted_requests.append(request) d_replica_scheduler._request_queue.append(request) @@ -226,17 +249,28 @@ def on_batch_end(self, batch: Batch) -> None: # Called when a batch finishes ex # vllm 和 sarathi的 free方法 和 orca的 free 方法不同 # vllm and sarathi free methods differ from orca's free method self.free(request.id) + + # 在移除请求之前，先计算当前的kvcache使用情况 + logger.debug(f"Before removing request {request.id}:") + self.replica.get_remaining_kv_cache_capacity() + # 移除请求 留给batch_end_event.py去做 + # self.replica.pending_requests.remove(request) + # 移除请求后释放相应的显存 + self.replica.release_request_kv_cache_memory(request) + logger.debug(f"Request {request.id} removed from replica and GPU memory released") + self.replica.get_remaining_kv_cache_capacity() + + + elif request.is_prefill_complete == True: self._preempted_requests.append(request) elif request.is_prefill_complete == False: - assert request.decode_arrived_at == float("inf"), "> debug" + assert request.decode_arrived_at == float("inf"), "decode_arrived_at must be infinity for incomplete prefill" - # > 用orca写一版get next batch。 - # > implement get next batch using orca approach. - # @abstractmethod # Mark as abstract method, requiring subclass implementation + # Implement get next batch using orca approach. def _get_next_batch(self) -> Batch: """ @@ -249,15 +283,10 @@ def _get_next_batch(self) -> Batch: Return: List of preempted tasks, List of new tasks """ - # print(f"> Debug: key: sw replica scheduler is forming batch _get_next_batch") - - requests = [] # Store requests to be processed in this batch num_tokens = [] # Store token counts for corresponding requests num_batch_tokens = 0 # Total tokens in current batch - # print(f"> Debug: entering _get_next_batch replica_id={self.replica._id} replica type={self.replica.replica_type}") - if self.replica.replica_type == ReplicaType.MIXED: pass elif self.replica.replica_type == ReplicaType.PREFILL: @@ -265,7 +294,6 @@ def _get_next_batch(self) -> Batch: # Request popping; original request popping didn't put them back, # batch； tmp_requests_to_remove = list() # Record requests to be removed from queue - # print(f"> Debug: entering PREFILL replica path len(self._request_queue)={len(self._request_queue)} len(self._preempted_requests)={len(self._preempted_requests)} ") # 对于batch end 加回来的请求（默认之前的放得下）# （没完成） # 因此 batch end 不能把完成p的request 放回p replica； 但可以放到 d replica中； 不过目前逻辑不需要放入d replica中 # For requests added back by batch end (assuming previous ones fit) # (not completed) # Therefore batch end cannot put completed p requests back to p replica; but can put them in d replica; however current logic doesn't require putting them in d replica @@ -278,37 +306,27 @@ def _get_next_batch(self) -> Batch: requests.append(request) # Add to request list num_tokens.append(next_num_tokens) # Record token count - # TODO: > > 能过去才能把kvcache 传递过去； 逻辑需要check一下；一些极端case，kvcache和req目前在d那边排队， 先传后排； 有可能实际是先排队 然后p2p传输 - # TODO: > > 有空再做：显存池； 判断大家的空间够不够； - # TODO: > > can only pass if space available; logic needs checking; for extreme cases, kvcache and req currently queued at d side, transmitted out of order; might actually queue first then p2p transmit - # TODO: > > do when free: GPU memory pool; check if everyone has enough space; + # TODO(tianhao909): implement GPU memory pool with space validation for KV cache transfer + # TODO(tianhao909): 实现显存池，判断空间是否足够再传递 KV cache + # TODO(tianhao909): handle extreme cases where kvcache queued at decode side before p2p transfer # For unprocessed requests; for request in self._request_queue: - # print(f"> Debug: entering PREFILL replica path _get_next_batch req id ={request.id} still in _req_queue, req.is_prefill_complete={request.is_prefill_complete} req type={request.request_type}, replica_id={self.replica._id} replica type={self.replica.replica_type}" ) - # print(f"> Debug: request id ={request.id} , request_type={request.request_type} _arrived_at={request._arrived_at} num_processed_tokens={request.num_processed_tokens} _latest_iteration_completed_at={request._latest_iteration_completed_at} _latest_iteration_scheduled_at={request._latest_iteration_scheduled_at}") if request.request_type == RequestType.PREFILL and request.is_prefill_complete == False: - # print(f"> Debug: entering PREFILL replica path num_batch_tokens={num_batch_tokens} next_num_tokens={next_num_tokens} max_tokens_in_batch={self._config.max_tokens_in_batch}") # 组batch + 判断到达时间 # Form batch + Check arrival time - # print(f"> Debug: request id ={request.id} , request_type={request.request_type} _arrived_at={request._arrived_at} num_processed_tokens={request.num_processed_tokens} _latest_iteration_completed_at={request._latest_iteration_completed_at} _latest_iteration_scheduled_at={request._latest_iteration_scheduled_at}") - next_num_tokens = self._get_request_next_num_tokens(request) # Get next token count needed by this request assert next_num_tokens == request.num_prefill_tokens if num_batch_tokens + next_num_tokens > self._config.max_tokens_in_batch: # If total batch tokens plus current request tokens exceed limit - # print(f"> Debug: break1: num_batch_tokens={num_batch_tokens} next_num_tokens={next_num_tokens} max_tokens_in_batch={self._config.max_tokens_in_batch}") - # print(f"> Debug: break1: investigate why {self.replica._id} prefill replica has request pileup, can't form batch for request {request._id}") break if len(self._allocation_map) == self._config.batch_size_cap: # If allocation map size reaches batch capacity limit - # print(f"> Debug: break2: investigate why {self.replica._id} prefill replica has request pileup, can't form batch for request {request._id}") break if len(requests) == self._max_micro_batch_size: # If request list size reaches maximum micro-batch size - # print(f"> Debug: break3: investigate why {self.replica._id} prefill replica has request pileup, can't form batch for request {request._id}") break # vllm sarathi method @@ -317,19 +335,23 @@ def _get_next_batch(self) -> Batch: # orca method if not self.can_allocate(self._max_blocks_per_sequence): - # print(f"> Debug: break4: investigate why {self.replica._id} prefill replica has request pileup, can't form batch for request {request._id}") break - # > pop（0） 不能写在遍历队列的循环里面， 遍历完之后再写 - # > pop(0) cannot be written inside queue iteration loop, write after iteration completes + # pop(0) cannot be written inside queue iteration loop, write after iteration completes # request = self._request_queue.pop(0) # Remove and get first request from request queue - # >: vllm and sarathi allocation approach in vidur: + # vllm and sarathi allocation approach in vidur: # self._allocate_request(request) # Allocate request resources - # >: orca allocation approach in vidur: allocate maximum blocks for request + # orca allocation approach in vidur: allocate maximum blocks for request self.allocate(request.id, self._max_blocks_per_sequence) + # self.replica.allocate_request_kv_cache_memory(request, self._max_blocks_per_sequence) + # Pass block_size to correctly convert blocks to tokens + # 传入 block_size，正确将 blocks 转为 tokens + self.replica.allocate_request_kv_cache_memory( + request, self._max_blocks_per_sequence, self._config.block_size) + requests.append(request) # Add request to request list tmp_requests_to_remove.append(request) @@ -338,29 +360,20 @@ def _get_next_batch(self) -> Batch: num_batch_tokens += next_num_tokens # Update total batch tokens elif request.request_type == RequestType.DECODE: - # assert request.request_type == RequestType.DECODE, "> Debug: shouldn't have already popped" continue if not requests: - # print(f"> Debug: failed to form prefill batch self.replica.replica_type={self.replica.replica_type} self.replica._id={self.replica._id} self._replica_id={self._replica_id}, req count={len(requests)}, num_tokens={num_tokens}") return else: - # assert len(tmp_requests_to_remove) == len(requests) ,'> Debug: popped and appended lengths must match' # 遍历完成后，从_request_queue中移除已处理的请求 # After iteration completes, remove processed requests from _request_queue for request in tmp_requests_to_remove: self._request_queue.remove(request) - # 看batch id 里面的req id ； - # Check batch id for req ids; - # for req in requests: - # print(f"> Debug: req id = {req._id}") - # print(f"> Debug: formed prefill batch self.replica.replica_type={self.replica.replica_type} self.replica._id={self.replica._id} self._replica_id={self._replica_id}, req count={len(requests)}, num_tokens={num_tokens}") return Batch(self._replica_id, requests, num_tokens) # Create and return Batch object elif self.replica.replica_type == ReplicaType.DECODE: tmp_requests_to_remove = list() - # print(f"> Debug: entering DECODE replica path len(self._request_queue)={len(self._request_queue)} len(self._preempted_requests)={len(self._preempted_requests)}") # 对于batch end 加回来的请求（默认之前的放得下）# （没完成） # 因此 batch end 不能把完成p的request 放回p replica； 但可以放到 d replica中； 不过目前逻辑不需要放入d replica中 # For requests added back by batch end (assuming previous ones fit) # (not completed) # Therefore batch end cannot put completed p requests back to p replica; but can put them in d replica; however current logic doesn't require putting them in d replica @@ -373,9 +386,7 @@ def _get_next_batch(self) -> Batch: requests.append(request) # Add to request list num_tokens.append(next_num_tokens) # Record token count - # assert len(self._request_queue) == 0, "> debug, > cannot let decode replica have reqs in _request_queue initially" for request in self._request_queue: - # print(f"> Debug: entering DECODE replica path _get_next_batch req id ={request.id} still in _req_queue, req.is_prefill_complete={request.is_prefill_complete} req type={request.request_type}, replica_id={self.replica._id} replica type={self.replica.replica_type}" ) if request.request_type == RequestType.PREFILL: continue @@ -386,22 +397,21 @@ def _get_next_batch(self) -> Batch: if request.is_prefill_complete == True: # 组batch + 判断到达时间 # Form batch + Check arrival time - assert request.decode_arrived_at != float('inf'), "> Debug: check if decode_arrived_at timing has been properly modified" + assert request.decode_arrived_at != float('inf'), "decode_arrived_at must be set before decode batching" # if request.decode_arrived_at == float('inf'): # continue - # > vllm sarathi orca都是这个 获取该请求下一次需要的token数量 - # > vllm sarathi orca all use this to get next token count needed by request + # vllm sarathi orca all use this to get next token count needed by request next_num_tokens = self._get_request_next_num_tokens(request) # decode next_num_tokens can only be 1 - assert next_num_tokens == 1, "> Debug: decode next_num_tokens must be 1" + assert next_num_tokens == 1, "decode next_num_tokens must be 1" # 如果批处理token总数加上当前请求token数超过最大限制 # If total batch tokens plus current request tokens exceed limit if num_batch_tokens + next_num_tokens > self._config.max_tokens_in_batch: - print(f"> Debug: break: num_batch_tokens + next_num_tokens > self._config.max_tokens_in_batch") + logger.debug("break: num_batch_tokens + next_num_tokens > max_tokens_in_batch") break # sarathi、vllm的方法：如果分配映射大小达到批处理容量上限 @@ -428,6 +438,13 @@ def _get_next_batch(self) -> Batch: # orca的方法： 为请求分配最大块数的资源 # orca method: allocate maximum blocks for request self.allocate(request.id, self._max_blocks_per_sequence) + # Pass block_size to correctly convert blocks to tokens + # 传入 block_size，正确将 blocks 转为 tokens + self.replica.allocate_request_kv_cache_memory( + request, self._max_blocks_per_sequence, self._config.block_size) + + # self.replica.allocate_request_kv_cache_memory(request, self._max_blocks_per_sequence) + requests.append(request) # Add request to request list tmp_requests_to_remove.append(request) # Determine token count; prefill tokens @@ -438,29 +455,11 @@ def _get_next_batch(self) -> Batch: continue if requests: - # print(f"> Debug: formed decode batch replica_id={self._replica_id}, req count={len(requests)}, num_tokens={num_tokens}") - # assert len(tmp_requests_to_remove) == len(requests) ,' > popped and appended lengths must match' - # After iteration completes, remove processed requests from _request_queue # 遍历完成后，从_request_queue中移除已处理的请求 for request in tmp_requests_to_remove: self._request_queue.remove(request) - # Check batch id for req ids; - # for req in requests: - # print(f"> Debug: req id = {req._id}") - # print(f"> Debug: formed decode batch self.replica.replica_type={self.replica.replica_type} self.replica._id={self.replica._id} self._replica_id={self._replica_id}, req count={len(requests)}, num_tokens={num_tokens}") return Batch(self._replica_id, requests, num_tokens) # Create and return Batch object if not requests: - # print(f"> Debug: failed to form decode batch, checking self._preempted_requests queue replica_id={self._replica_id}, req count={len(requests)}, num_tokens={num_tokens}") - - return - - - - - - - - - \ No newline at end of file + return \ No newline at end of file diff --git a/vidur-alibabacloud/vidur/scheduler/replica_stage_scheduler/replica_stage_schduler.py b/vidur-alibabacloud/vidur/scheduler/replica_stage_scheduler/replica_stage_schduler.py index 6a153945..4bcafd6b 100644 --- a/vidur-alibabacloud/vidur/scheduler/replica_stage_scheduler/replica_stage_schduler.py +++ b/vidur-alibabacloud/vidur/scheduler/replica_stage_scheduler/replica_stage_schduler.py @@ -42,11 +42,10 @@ def on_schedule(self) -> Tuple[Batch, BatchStage, ExecutionTime]: self._is_busy = True batch = self._batch_queue.pop(0) - # 模拟micro-batch在PP stage上的执行 - # TODO: 这块接入simai - # Simulate micro-batch execution on PP stage - # TODO: Integrate with simai + # 模拟 micro-batch 在 PP stage 上的执行 + # TODO(tianhao909): integrate with SimAI for execution time prediction + # TODO(tianhao909): 接入 SimAI 获取执行时间预测 execution_time = self._execution_time_predictor.get_execution_time( batch, self._stage_id, diff --git a/vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py b/vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py index e769e7b1..312e5a38 100644 --- a/vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py +++ b/vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py @@ -1,24 +1,127 @@ from vidur.config import ReplicaConfig from vidur.entities.replica import Replica +from vidur.logger import init_logger from vidur.utils.param_counter import ParamCounter +logger = init_logger(__name__) + class MemoryPlanner: def __init__(self, replica_config: ReplicaConfig, replica: Replica) -> None: self._param_counter = ParamCounter(replica_config) self._replica = replica + self._replica_config = replica_config + # TODO(tianhao909): support FP8 precision quantization + # TODO(tianhao909): 支持 FP8 精度量化 + if self._replica_config.pd_p2p_comm_dtype == "fp8": + logger.debug(f"FP8 enabled, dtype={self._replica_config.pd_p2p_comm_dtype}") + self.use_fp8 = True + else: + logger.debug(f"FP8 disabled, dtype={self._replica_config.pd_p2p_comm_dtype}") + self.use_fp8 = False + self.tp = self._replica_config.tensor_parallel_size + self.ep = self._replica_config.expert_model_parallel_size - def _get_kv_cache_memory_per_layer_per_request(self) -> int: - return ( - 2 # 2 bytes per float - * 2 # one for key, one for value - * self._replica.attention_head_dim - * self._replica.kv_heads_per_tensor_parallel_worker - * self._replica.max_request_tokens + # refer to https://github.com/alibaba/InferSim/blob/main/kvcache/kvcache.py + def get_mha_kvcache_size(self, config, use_fp8): + """ + Calculate MHA/GQA KV cache size (bytes) for all layers. + 计算所有层的 MHA/GQA KV Cache 大小（字节） + + Args: + config: Model config object / 模型配置对象 + use_fp8: Whether to use FP8 precision / 是否使用 FP8 精度 + + Returns: + int: Total KV cache size in bytes / KV Cache 总大小（字节） + """ + # 2 for K and V; layers * KV heads * head_dim + # 2 表示 K 和 V 两种缓存 + kvcache_size = ( + 2 * config.num_hidden_layers * config.num_key_value_heads * config.head_dim + ) + if not use_fp8: + # FP16/BF16 uses 2x bytes vs FP8 / FP16/BF16 比 FP8 多一倍字节 + kvcache_size *= 2 + return kvcache_size + + # refer to https://github.com/alibaba/InferSim/blob/main/kvcache/kvcache.py + # TODO(tianhao909): verify how TP splits MLA head_dim + # TODO(tianhao909): 核实 TP 如何切分 MLA 的 head_dim + def get_mla_kvcache_size(self, config, use_fp8): + """ + Calculate MLA KV cache size (bytes) for all layers. + 计算所有层的 MLA KV Cache 大小（字节） + + MLA uses kv_lora_rank + qk_rope_head_dim instead of full KV heads. + MLA 使用 kv_lora_rank + qk_rope_head_dim 代替完整 KV 头 + + Args: + config: Model config object / 模型配置对象 + use_fp8: Whether to use FP8 precision / 是否使用 FP8 精度 + + Returns: + int: Total KV cache size in bytes / KV Cache 总大小（字节） + """ + kvcache_size = config.num_hidden_layers * ( + config.kv_lora_rank + config.qk_rope_head_dim ) + if not use_fp8: + # FP16/BF16 uses 2x bytes vs FP8 / FP16/BF16 比 FP8 多一倍字节 + kvcache_size *= 2 + return kvcache_size + + # TODO(tianhao909): re-verify KV cache calc for DeepSeek/Qwen3 + # TODO(tianhao909): 重新核实 DeepSeek/Qwen3 的 KV Cache 计算方法 + # TODO(tianhao909): use per-request prefill/decode seq_len for PD separation + # TODO(tianhao909): PD 分离时传入不同 req 的 prefill_input_seq_len 和 decode_output_seq_len + # TODO(tianhao909): optimize from static to dynamic token allocation + # TODO(tianhao909): 目前按 max_request_tokens 静态分配，后续可优化为动态分配 + def _get_kv_cache_memory_per_layer_per_request(self) -> int: + """Calculate KV cache memory per layer per request (bytes). + 计算每层每请求的 KV Cache 内存（字节）""" + # Currently only DeepSeek-671B uses MLA KV cache + # 当前仅 DeepSeek-671B 使用 MLA KV Cache + if self._replica_config.model_name in ['deepseek-671B']: + kvcache_size_all_layers_per_token = self.get_mla_kvcache_size(self._replica_config.model_config, self.use_fp8) + kvcache_size_per_layer_per_token = kvcache_size_all_layers_per_token / self._replica_config.model_config.num_hidden_layers + prefill_input_seq_len = 1 + decode_output_seq_len = 1 + kvcache_size_per_layer_per_req = kvcache_size_per_layer_per_token * (prefill_input_seq_len + decode_output_seq_len) + return kvcache_size_per_layer_per_req + elif self._replica_config.model_name in ['qwen3-moe-235B', 'qwen3-next-80B']: + kvcache_size_all_layers_per_token = self.get_mha_kvcache_size(self._replica_config.model_config, self.use_fp8) + kvcache_size_per_layer_per_token = kvcache_size_all_layers_per_token / self._replica_config.model_config.num_hidden_layers + prefill_input_seq_len = 1 + decode_output_seq_len = 1 + kvcache_size_per_layer_per_req = kvcache_size_per_layer_per_token * (prefill_input_seq_len + decode_output_seq_len) + return kvcache_size_per_layer_per_req + + else: + # TP shard fallback / TP 切分回退 + # self._kv_heads_per_tensor_parallel_worker = ceil( + # self._model_config.num_kv_heads / self._replica_config.tensor_parallel_size + # ) + + return ( + 2 # 2 bytes per float + * 2 # one for key, one for value + * self._replica.attention_head_dim + * self._replica.kv_heads_per_tensor_parallel_worker + * self._replica.max_request_tokens + ) + # TODO(tianhao909): split get_num_parameters_per_device for P and D replicas + # TODO(tianhao909): 也要 P 和 D 分开适配，分开获得 get_num_parameters_per_device def _get_parameter_memory_per_device(self) -> int: - return 2 * self._param_counter.get_num_parameters_per_device() + """Get model parameter memory per device (bytes or GB for new models). + 获取每设备模型参数内存（字节，新模型返回 GB）""" + # New models return params in GB instead of count + # 三种新模型返回参数 GB 而不是参数个数 + if self._replica_config.model_name in ['deepseek-671B', 'qwen3-moe-235B', 'qwen3-next-80B']: + return self._param_counter.get_num_parameters_per_device() + else: + return 2 * self._param_counter.get_num_parameters_per_device() def _get_kv_cache_memory_per_device_per_request(self) -> int: return ( @@ -26,26 +129,183 @@ def _get_kv_cache_memory_per_device_per_request(self) -> int: ) def get_max_batch_size(self) -> int: - available_memory = ( + """ + Calculate maximum batch size based on GPU memory budget. + 根据 GPU 显存预算计算最大批处理大小 + + Formula / 计算公式: + 1. available_memory = total_GPU_memory * (1 - memory_margin_fraction) + 2. memory_for_kv_cache = available_memory - parameter_memory + 3. number_of_requests = memory_for_kv_cache // kv_cache_per_request + + For PD disaggregation / 对于 PD 分离架构: + - Prefill cluster: fewer params (larger EP), more KV cache memory + Prefill 集群: 参数量较小(EP较大), KV cache 可用内存较多 + - Decode cluster: more params (smaller EP), less KV cache memory + Decode 集群: 参数量较大(EP较小), KV cache 可用内存较少 + + Returns: + int: Maximum concurrent requests / 可同时处理的最大请求数 + """ + # ===== 1. Calculate available GPU memory / 计算GPU可用内存 ===== + available_memory_bytes = ( self._replica.total_memory_gb * 1024**3 * (1 - self._replica.memory_margin_fraction) ) - parameter_memory_per_device = self._get_parameter_memory_per_device() - kv_cache_memory_per_device_per_request = ( - self._get_kv_cache_memory_per_device_per_request() - ) + available_memory_gb = available_memory_bytes / (1024**3) + + if self._replica_config.model_name in ['deepseek-671B', 'qwen3-moe-235B', 'qwen3-next-80B']: + # ===== 2. Get model parameter memory (unit: Bytes) ===== + # Returns triple: (total_param_mem, prefill_param_mem, decode_param_mem) + total_param_mem, prefill_param_mem, decode_param_mem = self._get_parameter_memory_per_device() + + # ===== 3. Get per-request KV cache memory / 获取每请求KV cache内存 ===== + kv_cache_per_request = self._get_kv_cache_memory_per_device_per_request() - memory_for_kv_cache = available_memory - parameter_memory_per_device - number_of_requests = ( - memory_for_kv_cache // kv_cache_memory_per_device_per_request - ) + # ===== 4. Calculate KV cache available memory ===== + # Note: must use per-phase param memory, not total + # 注意: 必须使用各阶段各自的参数内存, PD分离下EP不同导致参数量不同 + prefill_kv_cache_memory = available_memory_bytes - prefill_param_mem # Prefill KV cache available + decode_kv_cache_memory = available_memory_bytes - decode_param_mem # Decode KV cache available + + # ===== 5. Calculate max supported requests ===== + # If KV cache memory is negative, model params exceed GPU memory + if prefill_kv_cache_memory > 0: + prefill_num_requests = int(prefill_kv_cache_memory // kv_cache_per_request) + else: + prefill_num_requests = 0 # OOM, set to 0 + + if decode_kv_cache_memory > 0: + decode_num_requests = int(decode_kv_cache_memory // kv_cache_per_request) + else: + decode_num_requests = 0 # OOM, set to 0 + + # ===== 6. 打印详细调试信息 | Print detailed debug info ===== + logger.info("\n" + "="*80) + logger.info("[MemoryPlanner] GPU Memory Allocation (GPU内存分配详情):") + logger.info("="*80) + logger.info(f" Total GPU mem (GPU总内存): {self._replica.total_memory_gb:.2f} GB") + logger.info(f" Mem margin (内存保留比例): {self._replica.memory_margin_fraction*100:.1f}%") + logger.info(f" Available mem (可用内存): {available_memory_gb:.2f} GB") + logger.info("-"*80) + logger.info(f" Total param mem (总参数内存): {total_param_mem / (1024**3):.2f} GB") + logger.info(f" Prefill param mem (Prefill参数内存): {prefill_param_mem / (1024**3):.2f} GB") + logger.info(f" Decode param mem (Decode参数内存): {decode_param_mem / (1024**3):.2f} GB") + logger.info("-"*80) + logger.info(f" Prefill KV cache avail (Prefill可用内存): {prefill_kv_cache_memory / (1024**3):.2f} GB") + logger.info(f" Decode KV cache avail (Decode可用内存): {decode_kv_cache_memory / (1024**3):.2f} GB") + logger.info(f" Per-req KV cache (每请求KV cache): {kv_cache_per_request / (1024**3):.6f} GB") + logger.info("-"*80) + logger.info(f" Prefill max reqs (Prefill最大请求数): {prefill_num_requests}") + logger.info(f" Decode max reqs (Decode最大请求数): {decode_num_requests}") + logger.info("="*80 + "\n") + + # ===== 7. OOM check and error handling / 内存不足检查与错误处理 ===== + # Check Prefill cluster memory / 检查Prefill集群内存 + if prefill_param_mem > available_memory_bytes: + logger.error(f"Prefill cluster OOM (Prefill集群内存不足)!") + logger.error(f" Param mem needed (需要参数内存): {prefill_param_mem / (1024**3):.2f} GB") + logger.error(f" Available mem (可用内存): {available_memory_gb:.2f} GB") + logger.error(f" Deficit (内存缺口): {(prefill_param_mem - available_memory_bytes) / (1024**3):.2f} GB") + logger.error(f"[Suggestion] Increase TP/EP, use larger GPU, or enable FP8") + + # Check Decode cluster memory / 检查Decode集群内存 + if decode_param_mem > available_memory_bytes: + logger.error(f"Decode cluster OOM (Decode集群内存不足)!") + logger.error(f" Param mem needed (需要参数内存): {decode_param_mem / (1024**3):.2f} GB") + logger.error(f" Available mem (可用内存): {available_memory_gb:.2f} GB") + logger.error(f" Deficit (内存缺口): {(decode_param_mem - available_memory_bytes) / (1024**3):.2f} GB") + logger.error(f"[Suggestion] Increase TP/EP, use larger GPU, or enable FP8") + + # Assert: at least one request must fit / 断言: 至少能处理一个请求 + assert prefill_num_requests > 0, ( + f"Prefill cluster OOM! param_mem({prefill_param_mem/(1024**3):.2f}GB) > " + f"available({available_memory_gb:.2f}GB), increase parallelism or use quantization" + ) + assert decode_num_requests > 0, ( + f"Decode cluster OOM! param_mem({decode_param_mem/(1024**3):.2f}GB) > " + f"available({available_memory_gb:.2f}GB), increase parallelism or use quantization" + ) + + # Return prefill max requests as system upper bound + # 返回Prefill的最大请求数 (作为系统上限) + return int(prefill_num_requests) + else: + parameter_memory_per_device = self._get_parameter_memory_per_device() + kv_cache_memory_per_device_per_request = ( + self._get_kv_cache_memory_per_device_per_request() + ) - assert ( - number_of_requests > 0 - ), "Not enough memory to store even a single request" + memory_for_kv_cache = available_memory_bytes - parameter_memory_per_device + number_of_requests = ( + memory_for_kv_cache // kv_cache_memory_per_device_per_request + ) + + logger.debug(f"available_memory={available_memory_gb}(GB) parameter_memory_per_device={parameter_memory_per_device / (1024**3)}(GB) memory_for_kv_cache={memory_for_kv_cache / (1024**3)} GB kv_cache_memory_per_device_per_request={kv_cache_memory_per_device_per_request / (1024**3)}(GB) number_of_requests={number_of_requests}") - return number_of_requests + assert ( + number_of_requests > 0 + ), "Not enough memory to store even a single request" + + return number_of_requests def get_max_request_slots(self) -> int: return self.get_max_batch_size() * self._replica.num_pipeline_stages + + def get_kv_cache_available_memory(self) -> int: + """ + Get actual available memory for KV cache (bytes). + 获取可用于 KV cache 的真实内存大小（字节） + + Formula / 计算公式: + kv_cache_available = GPU_available_memory - model_param_memory + + Returns: + int: Available memory for KV cache (bytes) / 可用于 KV cache 的内存 + """ + # ===== 1. Calculate available GPU memory (bytes) ===== + available_memory_bytes = ( + self._replica.total_memory_gb + * 1024**3 + * (1 - self._replica.memory_margin_fraction) + ) + + # ===== 2. Get model parameter memory ===== + # Previously used prefill_param_mem for all replicas incorrectly, + # causing decode replica KV cache available memory calculation error. + # Now select param memory based on replica type. + # 之前对所有 replica 都使用 prefill_param_mem， + # 导致 decode replica 的 KV cache 可用内存计算错误。 + # 现在根据 replica 类型选择对应的参数内存。 + if self._replica_config.model_name in ['deepseek-671B', 'qwen3-moe-235B', 'qwen3-next-80B']: + from vidur.entities.replica import ReplicaType + _, prefill_param_mem, decode_param_mem = self._get_parameter_memory_per_device() + + if hasattr(self._replica, 'replica_type') and self._replica.replica_type == ReplicaType.DECODE: + # Decode replica uses decode_param_mem + # Decode 副本使用 decode_param_mem + param_memory = decode_param_mem + logger.debug(f"get_kv_cache_available_memory: " + f"Decode replica uses decode_param_mem={decode_param_mem/(1024**3):.2f} GB " + f"(not prefill_param_mem={prefill_param_mem/(1024**3):.2f} GB)") + else: + # Prefill/Mixed replica uses prefill_param_mem + # Prefill/Mixed 副本使用 prefill_param_mem + param_memory = prefill_param_mem + logger.debug(f"get_kv_cache_available_memory: " + f"Prefill replica uses prefill_param_mem={prefill_param_mem/(1024**3):.2f} GB") + else: + param_memory = self._get_parameter_memory_per_device() + + # ===== 3. Calculate available KV cache memory ===== + kv_cache_available = available_memory_bytes - param_memory + + # Ensure non-negative / 确保非负 + if kv_cache_available < 0: + logger.warning(f"KV cache available is negative! param_mem({param_memory/(1024**3):.2f}GB) > available({available_memory_bytes/(1024**3):.2f}GB)") + assert kv_cache_available >= 0, f"kv_cache_available={kv_cache_available} must be >= 0" + kv_cache_available = 0 + + logger.info(f"[MemoryPlanner] Real KV cache available (真实KV cache可用内存): {kv_cache_available/(1024**3):.2f} GB") + return int(kv_cache_available) diff --git a/vidur-alibabacloud/vidur/simulator.py b/vidur-alibabacloud/vidur/simulator.py index af251876..4b5aede2 100644 --- a/vidur-alibabacloud/vidur/simulator.py +++ b/vidur-alibabacloud/vidur/simulator.py @@ -81,13 +81,15 @@ def run(self) -> None: # 设置系统时间为事件发生的时间 # Set system time to the event occurrence time self._set_time(event._time) - if tmp_pre_debug_time == 0 and event._time > tmp_pre_debug_time : + if tmp_pre_debug_time == 0 and event._time > tmp_pre_debug_time: tmp_pre_debug_time = event._time - elif tmp_pre_debug_time > 0 and tmp_pre_debug_time > event._time: - assert tmp_pre_debug_time <= event._time, f"> debug tmp_pre_debug_time={tmp_pre_debug_time} event._time={event._time}" + elif tmp_pre_debug_time > 0 and tmp_pre_debug_time > event._time: + assert tmp_pre_debug_time <= event._time, ( + f"Event time went backwards: prev={tmp_pre_debug_time} cur={event._time}" + ) - assert event._time >= 0, "> debug" - print(f"> Debug: len(_event_queue){len(self._event_queue)}, event_type={event._event_type} , time={event._time}") + assert event._time >= 0, "Event time must be non-negative" + logger.debug(f"len(_event_queue){len(self._event_queue)}, event_type={event._event_type}, time={event._time}") # 处理事件，事件可能会触发新的事件 # Handle the event, events may trigger new events @@ -106,6 +108,14 @@ def run(self) -> None: assert self._scheduler.is_empty() or self._terminate logger.info(f"Simulation ended at: {self._time}s") + + # [AICB优化] 模拟结束时打印缓存统计并保存查表 + try: + from vidur.entities.execution_time import _GLOBAL_AICB_CACHE + _GLOBAL_AICB_CACHE.print_stats() + _GLOBAL_AICB_CACHE.save_lookup_table() + except Exception as e: + print(f"[WARNING] Cannot print AICB cache stats (无法打印AICB缓存统计): {e}") def _write_output(self) -> None: logger.info("Writing output") @@ -136,7 +146,7 @@ def _init_event_queue(self) -> None: # 生成请求，把请求加入到时间队列中 # Generate requests and add them to the time queue for request in requests: - print(f"> Debug: arrived_at={request.arrived_at} 从 simulator的_init_event_queue() 生成 1个 RequestArrivalEvent, request_id={request._id}") + logger.debug(f"arrived_at={request.arrived_at} RequestArrivalEvent generated, request_id={request._id}") self._add_event(RequestArrivalEvent(request.arrived_at, request)) def _set_time(self, time: float) -> None: diff --git a/vidur-alibabacloud/vidur/types/device_sku_type.py b/vidur-alibabacloud/vidur/types/device_sku_type.py index f6f98b69..70696e1e 100644 --- a/vidur-alibabacloud/vidur/types/device_sku_type.py +++ b/vidur-alibabacloud/vidur/types/device_sku_type.py @@ -6,3 +6,6 @@ class DeviceSKUType(BaseIntEnum): A100 = 2 H100 = 3 H800 = 4 + H20 = 5 + H200 = 6 + GB200 = 7 diff --git a/vidur-alibabacloud/vidur/types/node_sku_type.py b/vidur-alibabacloud/vidur/types/node_sku_type.py index 19df26b0..8e86682b 100644 --- a/vidur-alibabacloud/vidur/types/node_sku_type.py +++ b/vidur-alibabacloud/vidur/types/node_sku_type.py @@ -8,3 +8,4 @@ class NodeSKUType(BaseIntEnum): A100_DGX = 4 H100_DGX = 5 H800_DGX = 6 + H20_DGX = 7 diff --git a/vidur-alibabacloud/vidur/utils/mfu_calculator.py b/vidur-alibabacloud/vidur/utils/mfu_calculator.py index fecab53a..8f3d8b13 100644 --- a/vidur-alibabacloud/vidur/utils/mfu_calculator.py +++ b/vidur-alibabacloud/vidur/utils/mfu_calculator.py @@ -1,13 +1,59 @@ from vidur.config import ReplicaConfig from vidur.entities import BatchStage +from vidur.entities.request import RequestType from vidur.utils.param_counter import ParamCounter +# MoE模型列表：这些模型需要区分 prefill/decode 的参数量 +# MoE model list: These models need separate prefill/decode parameter counts +MOE_MODELS_WITH_PD_SEPARATION = ['deepseek-671B', 'qwen3-moe-235B', 'qwen3-next-80B'] + + class MFUCalculator: + """ + MFU (Model FLOPs Utilization) 计算器 + 计算模型计算效率，支持 prefill/decode 分离场景 + + MFU (Model FLOPs Utilization) Calculator + Calculates model compute efficiency, supports prefill/decode separation scenarios + """ def __init__(self, replica_config: ReplicaConfig): + self._replica_config = replica_config + self._model_name = replica_config.model_name + + # 判断是否是需要区分 prefill/decode 的 MoE 模型 + # Determine if this is a MoE model requiring prefill/decode separation + self._is_pd_separated_model = self._model_name in MOE_MODELS_WITH_PD_SEPARATION + param_counter = ParamCounter(replica_config) - self._num_params_per_device = param_counter.get_num_parameters_per_device() + + # 根据模型类型获取参数量 + # Get parameter counts based on model type + if self._is_pd_separated_model: + # MoE模型：返回三元组 (total, prefill, decode) + # MoE model: returns tuple (total, prefill, decode) + params_result = param_counter.get_num_parameters_per_device() + self._num_params_per_device = params_result[0] # 总参数量 | Total params + self._prefill_num_params_per_device = params_result[1] # Prefill参数量 | Prefill params + self._decode_num_params_per_device = params_result[2] # Decode参数量 | Decode params + + # 打印重要信息便于检查 | Print important info for verification + print(f"[MFUCalculator] MoE model PD separation mode (MoE模型 PD分离模式)") + print(f"[MFUCalculator] model_name={self._model_name}") + print(f"[MFUCalculator] num_params_per_device (total)={self._num_params_per_device / 1024 / 1024 / 1024:.4f} GB") + print(f"[MFUCalculator] prefill_num_params_per_device={self._prefill_num_params_per_device / 1024 / 1024 / 1024:.4f} GB") + print(f"[MFUCalculator] decode_num_params_per_device={self._decode_num_params_per_device / 1024 / 1024 / 1024:.4f} GB") + else: + # 普通模型：返回单个值 + # Normal model: returns single value + self._num_params_per_device = param_counter.get_num_parameters_per_device() + self._prefill_num_params_per_device = self._num_params_per_device + self._decode_num_params_per_device = self._num_params_per_device + + print(f"[MFUCalculator] Normal model mode (普通模型模式)") + print(f"[MFUCalculator] model_name={self._model_name}") + print(f"[MFUCalculator] num_params_per_device={self._num_params_per_device}") model_config = replica_config.model_config @@ -20,9 +66,45 @@ def __init__(self, replica_config: ReplicaConfig): self._head_dimension = model_config.embedding_dim // model_config.num_q_heads self._device_flops = replica_config.device_config.fp16_tflops * 2**40 + def _get_batch_stage_type(self, batch_stage: BatchStage) -> RequestType: + """ + 获取 batch_stage 的类型（prefill 或 decode） + 通过检查第一个 request 的类型来判断 + + Get batch_stage type (prefill or decode) + Determined by checking the first request's type + """ + if not batch_stage.requests: + return RequestType.MIXED + # 假设同一个 batch_stage 中所有 request 类型相同 + # Assume all requests in the same batch_stage have the same type + return batch_stage.requests[0].request_type + def _get_mlp_flops(self, batch_stage: BatchStage) -> float: + """ + 计算 MLP 层的 FLOPs + 根据 batch_stage 类型选择对应的参数量 + + Calculate MLP layer FLOPs + Select corresponding parameter count based on batch_stage type + """ num_tokens = sum(batch_stage.num_tokens) - return 2 * num_tokens * self._num_params_per_device + + # 对于 MoE 模型，根据 stage 类型选择参数量 + # For MoE models, select parameter count based on stage type + if self._is_pd_separated_model: + stage_type = self._get_batch_stage_type(batch_stage) + if stage_type == RequestType.PREFILL: + params = self._prefill_num_params_per_device + elif stage_type == RequestType.DECODE: + params = self._decode_num_params_per_device + else: + # MIXED 类型使用总参数量 | MIXED type uses total params + params = self._num_params_per_device + else: + params = self._num_params_per_device + + return 2 * num_tokens * params def _get_attention_flops(self, batch_stage: BatchStage) -> float: total_flops = 0 @@ -42,5 +124,12 @@ def get_mfu(self, batch_stage: BatchStage) -> float: mlp_flops = self._get_mlp_flops(batch_stage) attention_flops = self._get_attention_flops(batch_stage) total_flops = mlp_flops + attention_flops + + # 防止除零错误：如果execution_time为0，返回0 + # Prevent division by zero: return 0 if execution_time is 0 + if batch_stage.execution_time == 0: + print(f"[WARNING] batch_stage.execution_time is 0, returning MFU as 0") + return 0.0 + total_flops_per_second = total_flops / batch_stage.execution_time return total_flops_per_second * 100 / self._device_flops diff --git a/vidur-alibabacloud/vidur/utils/param_counter.py b/vidur-alibabacloud/vidur/utils/param_counter.py index 5ef348f0..af678868 100644 --- a/vidur-alibabacloud/vidur/utils/param_counter.py +++ b/vidur-alibabacloud/vidur/utils/param_counter.py @@ -1,12 +1,18 @@ from math import ceil from vidur.config import ReplicaConfig +from vidur.logger import init_logger +import os +import json + +logger = init_logger(__name__) class ParamCounter: def __init__(self, replica_config: ReplicaConfig) -> None: self._replica_config = replica_config self._model_config = self._replica_config.model_config + self.config = self._model_config assert ( self._model_config.num_q_heads % self._replica_config.tensor_parallel_size @@ -34,6 +40,114 @@ def __init__(self, replica_config: ReplicaConfig) -> None: self._kv_heads_per_tensor_parallel_worker = ceil( self._model_config.num_kv_heads / self._replica_config.tensor_parallel_size ) + + # TODO(tianhao909): support FP8 precision quantization + # TODO(tianhao909): 支持 FP8 精度量化 + if self._replica_config.pd_p2p_comm_dtype == "fp8": + logger.debug(f"FP8 enabled, dtype={self._replica_config.pd_p2p_comm_dtype}") + self.use_fp8 = True + else: + logger.debug(f"FP8 disabled, dtype={self._replica_config.pd_p2p_comm_dtype}") + self.use_fp8 = False + self.tp = self._replica_config.tensor_parallel_size + self.ep = self._replica_config.expert_model_parallel_size + + # 标记是否已经打印过调试信息 | Flag to track if debug info has been printed + self._debug_printed = False + + if self._replica_config.model_name in ['deepseek-671B', 'qwen3-moe-235B', 'qwen3-next-80B']: + self.model_config_postprocessing() + # self._model_config + + def model_config_postprocessing(self, ): + # 初始化配置字典 | Initialize configuration dictionary + d = dict() + if self._replica_config.model_name == 'deepseek-671B': + # 使用相对路径定位配置文件 | Use relative path to locate config file + config_path = os.path.join(os.path.dirname(__file__), "..", "..", "data", "hf_configs", "deepseek_v3_config.json") + elif self._replica_config.model_name == 'qwen3-moe-235B': + # TODO(tianhao909): add corresponding JSON config file for Qwen3-MoE + # TODO(tianhao909): 增加对应的 JSON 配置文件 + config_path = os.path.join(os.path.dirname(__file__), "..", "..", "data", "hf_configs", "qwen3_moe_config.json") + elif self._replica_config.model_name == 'qwen3-next-80B': + config_path = os.path.join(os.path.dirname(__file__), "..", "..", "data", "hf_configs", "qwen3-next-80B-A3B_config.json") + logger.debug(f"config_path={config_path}") + # 检查配置文件是否存在 | Check if config file exists + if not os.path.exists(config_path): + logger.warning(f"Config file {config_path} not found, using default config") + return + # 以只读模式加载JSON配置 | Load JSON config in read-only mode + with open(config_path, "r") as f: + d = json.load(f) + + # 模型隐藏层维度 | Model hidden size dimension + self._model_config.hidden_size = d["hidden_size"] + # 隐藏层数量 | Number of hidden layers + self._model_config.num_hidden_layers = d["num_hidden_layers"] + + # 判断是否使用混合注意力机制（全注意力与线性注意力交替） + # Determine if using hybrid attention (alternating full and linear attention) + self._model_config.is_hybrid_linear = d.get("full_attention_interval") is not None + if self._model_config.is_hybrid_linear: + # 全注意力层数量：每隔N层插入一次 | Full attention layers: inserted every N layers + self._model_config.num_full_attn_layers = ( + self._model_config.num_hidden_layers // d["full_attention_interval"] + ) + # 线性注意力层数量：总层数减去全注意力层数 | Linear attention layers: total - full attention + self._model_config.num_linear_attn_layers = ( + self._model_config.num_hidden_layers - self._model_config.num_full_attn_layers + ) + # 线性注意力卷积核维度 | Linear attention convolution kernel dimension + self._model_config.linear_conv_kernel_dim = d["linear_conv_kernel_dim"] + # 线性注意力键向量头维度 | Linear attention key head dimension + self._model_config.linear_key_head_dim = d["linear_key_head_dim"] + # 线性注意力键向量头数 | Linear attention number of key heads + self._model_config.linear_num_key_heads = d["linear_num_key_heads"] + # 线性注意力值向量头维度 | Linear attention value head dimension + self._model_config.linear_value_head_dim = d["linear_value_head_dim"] + # 线性注意力值向量头数 | Linear attention number of value heads + self._model_config.linear_num_value_heads = d["linear_num_value_heads"] + + self._model_config.attn_type = "MHA/GQA" # Default attention: MHA or GQA / 默认注意力类型 + if "kv_lora_rank" in d: # If kv_lora_rank present, use MLA attention / 如果配置中包含 kv_lora_rank，则使用MLA + self._model_config.attn_type = "MLA" + + # Attention mechanism parameter setup / 注意力机制相关参数设置 + if self._model_config.attn_type == "MHA/GQA": # MHA/GQA type + self._model_config.num_attention_heads = d["num_attention_heads"] # Number of attention heads / 注意力头数量 + self._model_config.num_key_value_heads = d["num_key_value_heads"] # KV heads for GQA / 键和值的头数量 + if "head_dim" in d: # If head_dim specified in config + self._model_config.head_dim = d["head_dim"] + else: + self._model_config.head_dim = self._model_config.hidden_size // self._model_config.num_attention_heads # Compute from hidden_size / heads + elif self._model_config.attn_type == "MLA": # MLA type + self._model_config.q_lora_rank = d["q_lora_rank"] # Query LoRA rank / 查询向量LoRA的秩 + self._model_config.qk_nope_head_dim = d["qk_nope_head_dim"] # QK no-position head dim / 无位置编码的QK头维度 + self._model_config.qk_rope_head_dim = d["qk_rope_head_dim"] # QK RoPE head dim / 使用RoPE编码的QK头维度 + self._model_config.kv_lora_rank = d["kv_lora_rank"] # KV LoRA rank / 键值对LoRA的秩 + self._model_config.num_attention_heads = d["num_attention_heads"] # Total attention heads / 注意力头总数 + self._model_config.v_head_dim = d["v_head_dim"] # Value head dim / 值向量每个头的维度 + self._model_config.qk_head_dim = self._model_config.qk_nope_head_dim + self._model_config.qk_rope_head_dim # QK total head dim = nope + rope + + # FFN/MoE (Feed-Forward Network / Mixture of Experts) configuration + # FFN/MoE（前馈网络/专家混合模型）配置 + self._model_config.is_moe = True # Default enable MoE / 默认启用MoE + if "num_routed_experts" in d: # Routed expert count / 路由专家数量 + self._model_config.num_routed_experts = d["num_routed_experts"] + elif "num_experts" in d: # Fallback to num_experts field + self._model_config.num_routed_experts = d["num_experts"] + else: + self._model_config.is_moe = False # No MoE if no expert fields / 不使用MoE + self._model_config.num_routed_experts = 1 # Single expert (standard FFN) / 单一专家 + + if self._model_config.is_moe: # If MoE enabled / 如果启用了MoE + self._model_config.num_experts_per_tok = d["num_experts_per_tok"] # Experts activated per token / 每个token激活的专家数 + self._model_config.intermediate_size = d["moe_intermediate_size"] # Per-expert intermediate size / 每个专家的中间层大小 + self._model_config.num_shared_experts = d.get("num_shared_experts", 0) # Shared expert count / 共享专家数量 + else: # Standard FFN (no MoE) / 未启用MoE + self._model_config.num_experts_per_tok = 1 # Single "expert" / 标准FFN + self._model_config.intermediate_size = d["intermediate_size"] # Standard FFN intermediate size / 标准FFN中间层大小 + self._model_config.num_shared_experts = 0 # No shared experts / 无共享专家 def get_num_parameters_per_layer(self) -> int: num_parameters = 0 @@ -69,7 +183,346 @@ def get_num_parameters_per_layer(self) -> int: ) return num_parameters + + # Layer Dimension + # First 3 layers are dense (no gate). Based on the above calculation, + # each of the first 3 layers of DeepSeek V3 has parameter count: + # Layer 维度 + # 前 3 层是 dense，没有 gate，基于上面的计算，DeepSeek V3 前 3 层每层的参数量是： + # (单层MLA中Q的LoRA参数量48,760,320 + 单层MLA中KV的LoRA参数量20,906,496 + 单层 MLA中WO的参数量117,440,512 + （pre+post）attention layernorm的参数14336（即7168+7168）） + （每个专家的参数量44,040,192 * 9 （9 因为前 3 层 dense，每层固定激活8个路由专家和一个共享专家）） + # (48,760,320 + 20,906,496 + 117,440,512 + 14336) + (44,040,192 * 9) = 583,483,392 + # Last 58 layers are MoE sparse-activated experts. DeepSeek V3 per-layer params: + # 后 58 层是 MoE 稀疏激活专家，基于上面的计算，DeepSeek V3 后 58 层每层的参数量是： + # (48,760,320 + 20,906,496 + 117,440,512 + 14336) + (44,040,192 * 257 + 1,835,264) = 11,507,286,272 + # (单层MLA中Q的LoRA参数量48,760,320 + 单层MLA中KV的LoRA参数量20,906,496 + 单层 MLA中WO的参数量117,440,512 + （pre+post）attention layernorm的参数14336（即7168+7168）） + （每个专家的参数量44,040,192 * 257 （256个路由专家和一个共享专家） + 路由 Gate 的参数量1,835,264） + def get_num_parameters_per_layer_by_layer_id(self, layer_id: int = 0) -> tuple: + """ + Get parameter count per layer by layer_id. + Returns tuple: (params_per_layer, prefill_params_per_layer, decode_params_per_layer) + + 根据 layer_id 获取每层的参数量 + 返回三元组: (params_per_layer, prefill_params_per_layer, decode_params_per_layer) + """ + # 初始化变量 | Initialize variables + params_per_layer_per_gpu = 0 + prefill_params_per_layer_per_gpu = 0 + decode_params_per_layer_per_gpu = 0 + + if self._replica_config.model_name == 'deepseek-671B': + # 仅在首次调用时打印调试信息 | Only print debug info on first call + if not self._debug_printed: + logger.info("{s:{c}^{n}}".format(s="[ParamCounter] DeepSeek-671B Model Weights", n=60, c="-")) + attn_params_bytes = self.get_attn_params_size(self._model_config, self.use_fp8) + expert_params_bytes = self.get_expert_params_size(self._model_config, self.use_fp8) + logger.info(f"[ParamCounter] One MLA params size (MB): {attn_params_bytes / 1024 / 1024:.2f}") + logger.info(f"[ParamCounter] One expert params size (MB): {expert_params_bytes / 1024 / 1024:.2f}") + logger.info(f"[ParamCounter] use_fp8={self.use_fp8}, tp={self.tp}, ep={self.ep}") + self._debug_printed = True + + if layer_id >= 0 and layer_id <= 2: + # 前 3 层是 dense，每层固定激活8个路由专家和1个共享专家 + # First 3 layers are dense, each layer activates 8 routed experts + 1 shared expert + params_per_layer_per_gpu = (self.get_mla_params_size(self._model_config, self.use_fp8)/self.tp + + self.get_expert_params_size(self._model_config, self.use_fp8) * (8 + 1) / self.tp) + prefill_params_per_layer_per_gpu = params_per_layer_per_gpu + decode_params_per_layer_per_gpu = params_per_layer_per_gpu + + elif layer_id >= 3 and layer_id <= 60: + # 后 58 层是 MoE 稀疏激活专家 + # Remaining 58 layers are MoE sparse activated experts + mla_params = self.get_mla_params_size(self._model_config, self.use_fp8) / self.tp + expert_params = self.get_expert_params_size(self._model_config, self.use_fp8) + + params_per_layer_per_gpu = mla_params + expert_params * (256/self.ep + 1) + prefill_params_per_layer_per_gpu = mla_params + expert_params * (256/self._replica_config.prefill_world_size + 1) + decode_params_per_layer_per_gpu = mla_params + expert_params * (256/self._replica_config.decode_world_size + 1) + + elif self._replica_config.model_name == 'qwen3-next-80B': + # 仅在首次调用时打印调试信息 | Only print debug info on first call + if not self._debug_printed: + logger.info("{s:{c}^{n}}".format(s="[ParamCounter] Qwen3-Next-80B Model Weights", n=60, c="-")) + full_attn_params_bytes = self.get_attn_params_size(self._model_config, self.use_fp8) + linear_attn_params_bytes = self.get_linear_attn_params_size(self._model_config, self.use_fp8) + expert_params_bytes = self.get_expert_params_size(self._model_config, self.use_fp8) + logger.info(f"[ParamCounter] One full attn params size (MB): {full_attn_params_bytes / 1024 / 1024:.2f}") + logger.info(f"[ParamCounter] One linear attn params size (MB): {linear_attn_params_bytes / 1024 / 1024:.2f}") + logger.info(f"[ParamCounter] One expert params size (MB): {expert_params_bytes / 1024 / 1024:.2f}") + logger.info(f"[ParamCounter] use_fp8={self.use_fp8}, tp={self.tp}, ep={self.ep}") + self._debug_printed = True + + full_attn_params_bytes = self.get_attn_params_size(self._model_config, self.use_fp8) + linear_attn_params_bytes = self.get_linear_attn_params_size(self._model_config, self.use_fp8) + expert_params_bytes = self.get_expert_params_size(self._model_config, self.use_fp8) + + # 基础参数: 专家网络部分 | Base params: expert network part + params_per_layer_per_gpu = expert_params_bytes * ( + self.config.num_shared_experts + self.config.num_routed_experts / self._replica_config.world_size + ) + prefill_params_per_layer_per_gpu = expert_params_bytes * ( + self.config.num_shared_experts + self.config.num_routed_experts / self._replica_config.prefill_world_size + ) + decode_params_per_layer_per_gpu = expert_params_bytes * ( + self.config.num_shared_experts + self.config.num_routed_experts / self._replica_config.decode_world_size + ) + + # 根据 layer_id 添加注意力层参数 | Add attention layer params based on layer_id + if layer_id % 4 == 3: # Full attention layers (e.g., layer 3, 7, 11...) + params_per_layer_per_gpu += full_attn_params_bytes / self.tp + prefill_params_per_layer_per_gpu += full_attn_params_bytes / self.tp + decode_params_per_layer_per_gpu += full_attn_params_bytes / self.tp + else: # Linear attention layers (e.g., layer 0, 1, 2, 4, 5, 6...) + params_per_layer_per_gpu += linear_attn_params_bytes / self.tp + prefill_params_per_layer_per_gpu += linear_attn_params_bytes / self.tp + decode_params_per_layer_per_gpu += linear_attn_params_bytes / self.tp + + elif self._replica_config.model_name == 'qwen3-moe-235B': + # 仅在首次调用时打印调试信息 | Only print debug info on first call + if not self._debug_printed: + logger.info("{s:{c}^{n}}".format(s="[ParamCounter] Qwen3-MoE-235B Model Weights", n=60, c="-")) + attn_params_bytes = self.get_mha_params_size(self._model_config, self.use_fp8) + expert_params_bytes = self.get_expert_params_size(self._model_config, self.use_fp8) + logger.info(f"[ParamCounter] One MHA params size (MB): {attn_params_bytes / 1024 / 1024:.2f}") + logger.info(f"[ParamCounter] One expert params size (MB): {expert_params_bytes / 1024 / 1024:.2f}") + logger.info(f"[ParamCounter] use_fp8={self.use_fp8}, tp={self.tp}, ep={self.ep}") + self._debug_printed = True + + # Qwen3-MoE-235B: 128个路由专家, 0个共享专家, MHA/GQA注意力, 没有dense层 + # Qwen3-MoE-235B: 128 routed experts, 0 shared experts, MHA/GQA attention, no dense layers + mha_params = self.get_mha_params_size(self._model_config, self.use_fp8) + expert_params = self.get_expert_params_size(self._model_config, self.use_fp8) + + params_per_layer_per_gpu = mha_params + expert_params * 128 + prefill_params_per_layer_per_gpu = mha_params/self.tp + expert_params * (128/self._replica_config.prefill_world_size) + decode_params_per_layer_per_gpu = mha_params/self.tp + expert_params * (128/self._replica_config.decode_world_size) + + return params_per_layer_per_gpu, prefill_params_per_layer_per_gpu, decode_params_per_layer_per_gpu def get_num_parameters_per_device(self) -> int: - num_parameters_per_layer = self.get_num_parameters_per_layer() - return num_parameters_per_layer * self._num_layers_per_pipeline_stage + # TODO(tianhao909): refactor per-layer param calculation with layer_id support + # TODO(tianhao909): 重构 get_num_parameters_per_device 支持按 layer_id 计算 + if self._replica_config.model_name in ['deepseek-671B', 'qwen3-moe-235B', 'qwen3-next-80B']: + # Reference: see ExecutionTime._get_block_execution_time_by_layer_id + # Need to get start/end layer_id for the current pipeline stage + # 参考 ExecutionTime._get_block_execution_time_by_layer_id 的实现 + # 需要获取当前pipeline stage的起始和结束layer id + # try: + + pipeline_stage_id = getattr(self, '_pipeline_stage_id', 0) + start_layer = pipeline_stage_id * self._num_layers_per_pipeline_stage + end_layer = start_layer + self._num_layers_per_pipeline_stage + logger.debug(f"pipeline_stage_id={pipeline_stage_id} num_layers_per_pipeline_stage={self._num_layers_per_pipeline_stage} start_layer={start_layer} end_layer={end_layer}") + + params_per_gpu = 0 + prefill_params_per_gpu = 0 # 修正变量名 | Fixed variable name + decode_params_per_gpu = 0 # 修正变量名 | Fixed variable name + for layer_id in range(start_layer, end_layer): + params_per_layer, prefill_params_per_layer, decode_params_per_layer = self.get_num_parameters_per_layer_by_layer_id(layer_id) + params_per_gpu += params_per_layer + prefill_params_per_gpu += prefill_params_per_layer + decode_params_per_gpu += decode_params_per_layer + + # params_per_gpu 单位是B | Unit is Bytes + params_per_gpu_gb = params_per_gpu / 1024 / 1024 / 1024 # Convert to GB / 转换为GB + prefill_params_per_gpu_gb = prefill_params_per_gpu / 1024 / 1024 / 1024 # Convert to GB / 转换为GB + decode_params_per_gpu_gb = decode_params_per_gpu / 1024 / 1024 / 1024 # Convert to GB / 转换为GB + logger.info("{:<40} {:<10.2f}".format("Per GPU params size (GB):", params_per_gpu_gb)) + logger.info("{:<40} {:<10.2f}".format("Prefill Per GPU params size (GB):", prefill_params_per_gpu_gb)) + logger.info("{:<40} {:<10.2f}".format("Decode Per GPU params size (GB) :", decode_params_per_gpu_gb)) + logger.info(f"Prefill: tp={self.tp} dp={self._replica_config._num_prefill_replicas} ep={self._replica_config.prefill_world_size} prefill_params_per_gpu_gb={prefill_params_per_gpu_gb} (GB)") + logger.info(f"Decode: tp={self.tp} dp={self._replica_config._num_decode_replicas} ep={self._replica_config.decode_world_size} decode_params_per_gpu_gb={decode_params_per_gpu_gb} (GB)") + assert self._replica_config._num_prefill_replicas % 1 == 0 and self._replica_config._num_decode_replicas % 1 == 0, "Prefill and Decode replicas must be integer" + + # # 计算每张GPU上的模型参数总量（包括共享专家和路由专家） + # params_per_gpu = attn_params_bytes + expert_params_bytes * ( + # self._model_config.num_shared_experts + # + self._model_config.num_routed_experts / self.ep + # ) + + # params_per_gpu = params_per_gpu / 1024 / 1024 / 1024 # 转换为GB + # params_per_gpu *= self._model_config.num_hidden_layers # 乘以层数得到总参数量 + # # 计算可用KV缓存内存（总显存减去模型参数、运行时开销和编码器预留） + # self.kvcache_mem = ( + # self.gpu.mem - params_per_gpu - 15 - 5 + # ) # 15GB for runtime, 5GB for encoder（15GB用于运行时，5GB用于编码器） + # print("{:<40} {:<10.2f}".format("Per GPU params size (GB):", params_per_gpu)) # 打印每GPU参数大小（GB） + + # Return tuple: (total params, prefill params, decode params) + # 返回三元组: (总参数量, prefill参数量, decode参数量) + return params_per_gpu, prefill_params_per_gpu, decode_params_per_gpu + + + # except AttributeError: + # # 如果无法获取_pipeline_stage_id，则回退到原来的实现 + # num_parameters_per_layer = self.get_num_parameters_per_layer() + # return num_parameters_per_layer * self._num_layers_per_pipeline_stage + else: + num_parameters_per_layer = self.get_num_parameters_per_layer() + return num_parameters_per_layer * self._num_layers_per_pipeline_stage + + def get_attn_params_size(self, config, use_fp8): + if config.attn_type == "MHA/GQA": # MHA or GQA attention type / MHA或GQA注意力类型 + return get_mha_params_size(self, config, use_fp8) + elif config.attn_type == "MLA": # MLA architecture / MLA结构 + return get_mla_params_size(self, config, use_fp8) + + + # Reference: /InferSim/params/params.py + # 参考自 /InferSim/params/params.py + # def get_mha_params_size(config: ModelConfig, use_fp8: bool): + def get_mha_params_size(self, config, use_fp8): + wq = config.hidden_size * config.num_attention_heads * config.head_dim # Q weight: hidden * heads * head_dim + wk = config.hidden_size * config.num_key_value_heads * config.head_dim # K weight: hidden * kv_heads * head_dim + wv = config.hidden_size * config.num_key_value_heads * config.head_dim # V weight: hidden * kv_heads * head_dim + wo = config.hidden_size * config.num_attention_heads * config.head_dim # Output weight: hidden * heads * head_dim + if use_fp8: # FP8 quantization / FP8量化 + return wq + wk + wv + wo # Single precision storage / 单精度存储 + return 2 * (wq + wk + wv + wo) # Full precision (e.g. FP16, 2 bytes per param) / 全精度 + + # MLA (suited for DeepSeek) / MLA（适合 DeepSeek） + # DeepSeek V3 parameter derivation references: + # dpsk v3 参数推导参考： + # https://zhuanlan.zhihu.com/p/21455638257 + # https://yangwenbo.com/articles/deepseek-v3-parameter-size.html + # "hidden_size": 7168, + # "num_key_value_heads": 128, + # "v_head_dim": 128, + # "kv_lora_rank": 512, + + # "num_attention_heads": 128, + # "q_lora_rank": 1536, + + # "qk_nope_head_dim": 128, + # "qk_rope_head_dim": 64, + + # "num_hidden_layers": 61, + # def get_mla_params_size(config: ModelConfig, use_fp8: bool): + def get_mla_params_size(self, config, use_fp8): + # Per-layer MLA Q LoRA params: + # 单层 MLA 中 Q 的 LoRA 参数量是： + # = 7168 * 1536 + 1536 + 1536 * 128 * (128 + 64) = 48,760,320 + # = wq_down + wq_up + # = (config.hidden_size * config.q_lora_rank) + (config.q_lora_rank * config.num_attention_heads * (config.qk_nope_head_dim + config.qk_rope_head_dim)) + # = (config.hidden_size * config.q_lora_rank) + (config.q_lora_rank * config.num_attention_heads * (config.qk_head_dim)) + wq_down = config.hidden_size * config.q_lora_rank # Q LoRA down-projection / Q的LoRA下投影矩阵参数量 + wq_up = config.q_lora_rank * config.num_attention_heads * config.qk_head_dim # Q LoRA up-projection / Q的LoRA上投影矩阵参数量 + # Per-layer MLA KV LoRA params: + # 单层 MLA 中 KV 的 LoRA 参数量是： + # = 7168 * (512 + 64) + 512 + 512 * 128 * (128 + 128) = 20,906,496 + # = wkv_down + 512 + wkv_up (TODO(tianhao909): clarify what the 512 constant represents) + # = config.hidden_size *（config.kv_lora_rank + config.qk_rope_head_dim) + 512 + config.kv_lora_rank * config.num_attention_heads * (config.qk_nope_head_dim + config.qk_rope_head_dim) + # = (config.hidden_size * config.kv_lora_rank) + (config.kv_lora_rank * config.num_key_value_heads * (config.qk_nope_head_dim + config.qk_rope_head_dim)) + wkv_down = config.hidden_size * config.kv_lora_rank # KV LoRA down-projection / KV的LoRA下投影矩阵参数量 + wkv_up = ( # KV LoRA up-projection / KV的LoRA上投影矩阵参数量 + config.kv_lora_rank + * config.num_attention_heads + * (config.qk_nope_head_dim + config.v_head_dim) + ) + # Per-layer MLA output (WO) params: + # 单层 MLA 中 WO 的参数量是 + # 128 * 128 * 7168 = 117,440,512 + # config.num_attention_heads * config.v_head_dim * config.hidden_size + wo = config.hidden_size * config.num_attention_heads * config.v_head_dim # Output weight / 输出权重参数量 + if use_fp8: # FP8 quantization / FP8量化 + return wq_down + wq_up + wkv_down + wkv_up + wo # Sum all params (single precision) / 返回所有参数之和 + # Unit: Bytes / 单位:B + return 2 * (wq_down + wq_up + wkv_down + wkv_up + wo) # FP16: multiply by 2 / 否则乘以2 + + # Additionally: pre+post attention layernorm params = 7168*2 = 14,336 + # DeepSeek V3 MLA total across 61 layers: + # 另外：pre+post attention layernorm 的参数量 = 7168*2 = 14,336 + # 所以 DeepSeek V3 的 MLA 部分共 61 层的总参数量是： + # (48,760,320 + 20,906,496 + 117,440,512 + 14,336) * 61 = 11,414,421,504 (~11B) + + + # def get_gdn_params_size(config: ModelConfig, use_fp8: bool): + def get_gdn_params_size(self, config, use_fp8): + wq = config.hidden_size * config.linear_num_key_heads * config.linear_key_head_dim # Q linear attention weight + wk = wq # K weight same as Q + wv = ( # V weight params + config.hidden_size + * config.linear_num_value_heads + * config.linear_value_head_dim + ) + wz = wv # Z weight same as V + wa = config.hidden_size * config.linear_num_value_heads # A gate params + wb = wa # B gate same as A + s = wq + wk + wv + wz + wa + wb # Total primary weight params + wconv = ( # Conv kernel weight part 1 + config.linear_num_key_heads + * config.linear_key_head_dim + * config.linear_conv_kernel_dim + ) + wconv += ( # Conv kernel weight part 2 + config.linear_num_key_heads + * config.linear_key_head_dim + * config.linear_conv_kernel_dim + ) + wconv += ( # Conv kernel weight part 3 + config.linear_num_value_heads + * config.linear_value_head_dim + * config.linear_conv_kernel_dim + ) + if use_fp8: # FP8 quantization + return s + wconv # Primary + conv params (single precision) + return 2 * s + wconv # Primary *2, conv stays single precision + + + # def get_attn_params_size(config: ModelConfig, use_fp8: bool): + def get_attn_params_size(self, config, use_fp8): + if config.attn_type == "MHA/GQA": # MHA or GQA attention type + return self.get_mha_params_size(config, use_fp8) + elif config.attn_type == "MLA": # MLA architecture + return self.get_mla_params_size(config, use_fp8) + + + # def get_linear_attn_params_size(config: ModelConfig, use_fp8: bool): + def get_linear_attn_params_size(self, config, use_fp8): + return self.get_gdn_params_size(config, use_fp8) # Get linear attention (GD-Nets style) params / 获取线性注意力参数总量 + + # MoE (suited for DeepSeek) / MoE（适合 DeepSeek） + # "num_hidden_layers": 61, + # "hidden_size": 7168, + # "moe_intermediate_size": 2048, // Routed expert MLP intermediate dim / 路由专家 MLP 的中间维度 + # "n_shared_experts": 1, // Shared expert count / 共享专家数量 + # "n_routed_experts": 256, // Routed expert count / 路由专家数量 + # "first_k_dense_replace": 3, // First K layers use dense instead of MoE / 前几层使用dense替换MoE + # "intermediate_size": 18432, // First 3 layers (9*moe_intermediate_size) / 前3层 + + # Per-expert params: / 每个专家的参数量是： + # 7168 * 2048 * 3 = 44,040,192 + # config.hidden_size * config.moe_intermediate_size * 3 + # Router gate params: / 路由 Gate 的参数量是： + # 256 * 7168 + 256 = 1,835,264 + # First 3 dense layers (8 routed + 1 shared per layer): / 前 3 层 dense（固定激活 8 路由专家）： + # 44,040,192 * 9 * 3 = 1,189,085,184 + # Last 58 sparse layers (dynamically activate 8 routed): / 后 58 层稀疏（动态激活 8 路由专家）： + # (44,040,192 * 257 + 1,835,264) * 58 = 656,569,547,264 + # DeepSeek V3 MoE total params: / DeepSeek V3 MoE 部分总参数量： + # 1,189,085,184 + 656,569,547,264 = 657,758,632,448 (~657B) + # Active params per forward (1 shared + 8 routed): / 每次计算激活参数量（1共享 + 8路由）： + # 44,040,192 * 9 * 61 + 1,835,264 * 58 = 24,284,510,720 (~24B) + # def get_expert_params_size(config: ModelConfig, use_fp8: bool): + def get_expert_params_size(self, config, use_fp8): + if self._replica_config.model_name in [ 'qwen3-moe-235B']: + # config.intermediate_size = 122888 + # config.moe_intermediate_size = 1536 + config.intermediate_size = config.moe_intermediate_size + w = 3 * config.hidden_size * config.intermediate_size # MoE expert FFN params (W1, W2, W3) / MoE专家前馈网络参数量 + else: + w = 3 * config.hidden_size * config.intermediate_size # MoE expert FFN params (W1, W2, W3) / MoE专家前馈网络参数量 + if not use_fp8: # Not using FP8 / 不使用FP8量化 + w *= 2 # Double for FP16 / 参数量翻倍 + return w # Return expert params total / 返回专家参数总量 + + + # def load_attn_weights_time(config: ModelConfig, use_fp8: bool, gpu: GPU): + def load_attn_weights_time(self, config, use_fp8, gpu): + size = self.get_attn_params_size(config, use_fp8) # Get attention weights size (bytes) / 获取注意力模块权重总大小 + return size / 1024 / 1024 / 1024 / gpu.mem_bw # Convert to GB / mem_bw = load time (s) / 转换为GB并除以GPU内存带宽 + + + # def load_moe_weights_time(config: ModelConfig, use_fp8: bool, gpu: GPU, num_gpus): + def load_moe_weights_time(self, config, use_fp8, gpu, num_gpus): + size = self.get_expert_params_size(config, use_fp8) # Get single expert weights size / 获取单个专家权重大小 + size *= config.num_routed_experts / num_gpus # Distribute across GPUs / 总专家数分配到多个GPU上 + return size / 1024 / 1024 / 1024 / gpu.mem_bw # Load time in seconds / 加载时间（秒） From 8f0571208f49d02d19ca1b648e44d058bd9bf3e7 Mon Sep 17 00:00:00 2001 From: tianhao909 <843101550@qq.com> Date: Tue, 14 Apr 2026 10:39:23 +0000 Subject: [PATCH 2/8] fix(vidur): address PR #268 Copilot review + add test report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [EN] - Comment out untested H200/GB200 device configs (device_sku_config.py, device_sku_type.py) - Clean pd_p2p_comm_dtype choices to tested types: fp8, float16, float32 - Fix .gitignore: add directory unignore before negation rules - Replace print() with logger calls (simulator.py, mfu_calculator.py, sklearn_predictor.py) - Fix duplicate write_image in metrics_store.py, keep _safe_write_image only - Add PD separation design comments in memory_planner.py - Fix float division to int division in kvcache calculation - Enrich TODO comments for seq_len=1 limitation and MFU semantic issue - Add AICB backend empirical formula explanation comments - Add Data Preparation section to README.md and README_CN.md - Add test tutorial and report (test_tutorial_report_cn.md) [ZH] - 注释掉未测试的 H200/GB200 设备配置 - 清理 pd_p2p_comm_dtype choices 仅保留已测试类型 - 修复 .gitignore negation 规则 - 将 print() 替换为 logger 调用 - 修复 metrics_store.py 重复 write_image，仅保留 _safe_write_image - 添加 PD 分离设计说明注释 - 修复 kvcache 计算中浮点除法为整除 - 丰富 seq_len=1 限制和 MFU 语义问题的 TODO 注释 - 添加 AICB 后端经验公式说明注释 - README 双语添加数据准备章节 - 添加测试教程和报告文档 --- vidur-alibabacloud/.gitignore | 1 + vidur-alibabacloud/README.md | 30 +++ vidur-alibabacloud/README_CN.md | 30 +++ .../tests/test_tutorial_report_cn.md | 206 ++++++++++++++++++ vidur-alibabacloud/vidur/config/config.py | 7 +- .../vidur/config/device_sku_config.py | 38 ++-- .../sklearn_execution_time_predictor.py | 10 +- .../vidur/metrics/metrics_store.py | 3 +- .../vidur/scheduler/utils/memory_planner.py | 28 ++- vidur-alibabacloud/vidur/simulator.py | 2 +- .../vidur/types/device_sku_type.py | 5 +- .../vidur/utils/mfu_calculator.py | 26 ++- 12 files changed, 346 insertions(+), 40 deletions(-) create mode 100644 vidur-alibabacloud/tests/test_tutorial_report_cn.md diff --git a/vidur-alibabacloud/.gitignore b/vidur-alibabacloud/.gitignore index 7efc924e..99e46767 100644 --- a/vidur-alibabacloud/.gitignore +++ b/vidur-alibabacloud/.gitignore @@ -196,6 +196,7 @@ examples/vidur-ali-scenarios/simulator_output/ SimAI/ core.* *.csv +!data/aicb_workload/ !data/aicb_workload/*.csv !data/aicb_workload/cache/ !data/aicb_workload/cache/*.csv diff --git a/vidur-alibabacloud/README.md b/vidur-alibabacloud/README.md index 2d6e5fe5..7af82c86 100644 --- a/vidur-alibabacloud/README.md +++ b/vidur-alibabacloud/README.md @@ -127,6 +127,36 @@ pip install -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ --- +### 5. Data Preparation + +The examples below use trace files from `data/processed_traces/`. These files are provided by the upstream [microsoft/vidur](https://github.com/microsoft/vidur) project. + +**Option A**: Clone upstream vidur and copy the trace files: + +```bash +git clone https://github.com/microsoft/vidur.git /tmp/vidur +cp -r /tmp/vidur/data/processed_traces ./data/ +``` + +**Option B**: If you already have the vidur data locally: + +```bash +cp -r /path/to/vidur/data/processed_traces ./data/ +``` + +After preparation, your directory structure should look like: + +``` +data/ +├── processed_traces/ +│ ├── splitwise_conv.csv +│ ├── splitwise_code.csv +│ └── arxiv_summarization_stats_llama2_tokenizer_filtered_v2.csv +└── hf_configs/ # Already included in this repo +``` + +--- + ## ▶️ Running Examples ### Run DeepSeek-671B with AICB diff --git a/vidur-alibabacloud/README_CN.md b/vidur-alibabacloud/README_CN.md index 372f8303..e37cc9ba 100644 --- a/vidur-alibabacloud/README_CN.md +++ b/vidur-alibabacloud/README_CN.md @@ -126,6 +126,36 @@ pip install -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ --- +### 5. 数据准备 + +下面的示例使用 `data/processed_traces/` 中的 trace 文件。这些文件来自上游 [microsoft/vidur](https://github.com/microsoft/vidur) 项目。 + +**方式一**：从上游 vidur 克隆并拷贝 trace 文件： + +```bash +git clone https://github.com/microsoft/vidur.git /tmp/vidur +cp -r /tmp/vidur/data/processed_traces ./data/ +``` + +**方式二**：如果本地已有 vidur 数据： + +```bash +cp -r /path/to/vidur/data/processed_traces ./data/ +``` + +准备完成后，目录结构应如下： + +``` +data/ +├── processed_traces/ +│ ├── splitwise_conv.csv +│ ├── splitwise_code.csv +│ └── arxiv_summarization_stats_llama2_tokenizer_filtered_v2.csv +└── hf_configs/ # 本仓库已包含 +``` + +--- + ## ▶️ 运行示例 ### 使用 AICB 运行 DeepSeek-671B diff --git a/vidur-alibabacloud/tests/test_tutorial_report_cn.md b/vidur-alibabacloud/tests/test_tutorial_report_cn.md new file mode 100644 index 00000000..cbfdb825 --- /dev/null +++ b/vidur-alibabacloud/tests/test_tutorial_report_cn.md @@ -0,0 +1,206 @@ +# Vidur-AlibabaCloud 测试教程与测试报告 + +> 测试日期: 2026-04-14 +> 测试环境: 8x NVIDIA H20-3e (143 GB each), Python 3.13.11 + +--- + +## 1. 环境安装步骤 + +### 1.1 Python 环境 + +```bash +# 使用 base conda 环境 (Python 3.10+) +conda activate base + +# 或创建专用环境 +conda env create -p ./env -f ./environment.yml +conda activate vidur +``` + +### 1.2 安装依赖 + +```bash +cd vidur-alibabacloud +pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ +pip install -r requirements-dev.txt -i https://mirrors.aliyun.com/pypi/simple/ +``` + +### 1.3 数据准备 + +```bash +# 从上游 microsoft/vidur 获取 trace 文件 +git clone https://github.com/microsoft/vidur.git /tmp/vidur +cp -r /tmp/vidur/data/processed_traces ./data/ + +# 从上游 microsoft/vidur 获取 profiling 数据 (Native Vidur 模式需要) +cp -r /tmp/vidur/data/profiling ./data/ +``` + +准备完成后目录结构： + +``` +data/ +├── processed_traces/ # trace 文件 (从 microsoft/vidur 拷贝) +│ ├── splitwise_conv.csv +│ ├── splitwise_code.csv +│ └── arxiv_summarization_stats_llama2_tokenizer_filtered_v2.csv +├── profiling/ # profiling 数据 (从 microsoft/vidur 拷贝, Native Vidur 模式需要) +│ ├── compute/ +│ └── network/ +├── hf_configs/ # 已包含在仓库中 +└── aicb_workload/ # 已包含在仓库中 +``` + +--- + +## 2. PD 分离单元测试 + +### 2.1 运行命令 + +```bash +cd vidur-alibabacloud +python -m pytest tests/test_pd_separation.py -v +``` + +### 2.2 测试结果 + +| 测试用例 | 状态 | 说明 | +|---------|------|------| +| `TestPDOff::test_config_init_defaults` | PASSED | PD 关闭时配置默认值验证 | +| `TestPDOff::test_mixed_mode_cluster` | PASSED | 混合模式集群创建验证 | +| `TestPDOn::test_pd_cluster_creation` | PASSED | PD 开启时集群创建验证 | +| `TestPDOn::test_per_phase_world_size` | PASSED | 各阶段 world size 计算验证 | +| `TestPDParamsFallback::test_none_fallback` | PASSED | 参数回退机制验证 | +| `TestPDParamsFallback::test_explicit_per_phase_params` | PASSED | 显式参数覆盖验证 | +| `TestIllegalPdNodeRatio::test_zero_ratio` | PASSED | 非法比例 0 拒绝验证 | +| `TestIllegalPdNodeRatio::test_negative_ratio` | PASSED | 非法负比例拒绝验证 | +| `TestIllegalPdNodeRatio::test_ratio_greater_than_one` | PASSED | 非法比例 >1 拒绝验证 | +| `TestNumPrefillReplicasPriority::test_explicit_prefill_replicas` | PASSED | 显式 prefill 副本数优先级验证 | + +**结果: 10/10 通过, 耗时 0.16s** + +--- + +## 3. 集成测试 — Llama-3-8B Native Vidur 模式 + +### 3.1 运行命令 + +```bash +cd vidur-alibabacloud + +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype float32 \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 10 \ + --length_generator_config_type trace \ + --trace_request_length_generator_config_max_tokens 2048 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --interval_generator_config_type poisson \ + --cluster_config_num_replicas 4 \ + --replica_config_pd_node_ratio 0.5 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name meta-llama/Meta-Llama-3-8B \ + --replica_config_tensor_parallel_size 4 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 1 \ + --random_forrest_execution_time_predictor_config_backend vidur +``` + +### 3.2 测试结果 + +| 项目 | 结果 | +|------|------| +| 状态 | **通过** | +| 模拟结束时间 | 4.059s | +| 处理请求数 | 10 | +| 集群配置 | 4 replicas, PD ratio=0.5 (2P+2D) | +| 输出文件 | request_metrics.csv, batch_metrics.csv, plots/ 等 | +| 注意事项 | PNG 生成跳过 (无 Chrome/Kaleido), CSV 数据正常保存 | + +### 3.3 前置依赖 + +- `data/processed_traces/splitwise_conv.csv` — 从 microsoft/vidur 拷贝 +- `data/profiling/` — 从 microsoft/vidur 拷贝 (Native Vidur 模式必需) + +--- + +## 4. 集成测试 — DeepSeek-671B AICB 模式 + +### 4.1 运行命令 + +```bash +cd vidur-alibabacloud + +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 5 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 1024 \ + --fixed_request_length_generator_config_decode_tokens 10 \ + --cluster_config_num_replicas 4 \ + --replica_config_pd_node_ratio 0.5 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name deepseek-671B \ + --replica_config_tensor_parallel_size 8 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_expert_model_parallel_size 8 \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --replica_config_device h20 +``` + +### 4.2 测试结果 + +| 项目 | 结果 | +|------|------| +| 状态 | **通过** | +| 模拟结束时间 | 0.038s | +| 处理请求数 | 5 | +| 集群配置 | 4 replicas, PD ratio=0.5 (2P+2D), H20 GPU | +| 已知警告 | "AICB data is empty, using default execution time" — 预期行为 | +| 注意事项 | README 示例使用 TP=2 EP=8 在 H20 上会 OOM, 需要 TP=8 + FP8 才能跑通 | + +### 4.3 GPU 内存分析 + +DeepSeek-671B 模型参数量极大, 在 H20 (141GB) 上需要较高的并行度: + +| 配置 | Prefill 参数内存 | 可用内存 | 状态 | +|------|-----------------|---------|------| +| TP=2, EP=8, FP16 | 320.97 GB | 72.00 GB | OOM | +| TP=4, EP=8, FP16 | 162.86 GB | 126.90 GB | OOM | +| TP=8, EP=8, FP8 | ~81 GB | 126.90 GB | **通过** | + +--- + +## 5. 测试结果汇总 + +| 测试类型 | 测试项 | 状态 | 说明 | +|---------|--------|------|------| +| Layer 1 | PD 分离单元测试 (10 cases) | **全部通过** | 无外部依赖 | +| Layer 2 | Llama-3-8B Native Vidur | **通过** | 需要 processed_traces + profiling 数据 | +| Layer 2 | DeepSeek-671B AICB (H20 FP8) | **通过** | 需要 H20 + TP=8 + FP8 配置 | +| Layer 3 | run_scenarios.sh 四场景 | **未测试** | 需要 AICB Docker 环境 | +| Layer 3 | Llama-3-8B SimAI Simulation | **未测试** | 需要编译 SimAI ns3 | +| Layer 3 | Llama-3-8B SimAI Analytical | **未测试** | 需要编译 SimAI Analytical | + +--- + +## 6. 已知限制与依赖 + +| 限制 | 说明 | +|------|------| +| profiling 数据 | Native Vidur 模式需要 `data/profiling/` (来自 microsoft/vidur) | +| AICB 数据 | DeepSeek-671B AICB 模式使用默认执行时间 (无实际 profiling 数据) | +| PNG 输出 | 需要 Chrome/Kaleido 才能生成 PNG 图表, 否则只输出 CSV | +| GPU 内存 | DeepSeek-671B 在 H20 上需要 TP>=8 + FP8 才能通过内存检查 | +| SimAI 构建 | SimAI Simulation/Analytical 模式需要额外编译步骤 | +| seq_len=1 | KV cache 计算中 seq_len 硬编码为 1 (已知限制, 见 TODO 注释) | diff --git a/vidur-alibabacloud/vidur/config/config.py b/vidur-alibabacloud/vidur/config/config.py index 10007218..3fe44e99 100644 --- a/vidur-alibabacloud/vidur/config/config.py +++ b/vidur-alibabacloud/vidur/config/config.py @@ -484,8 +484,11 @@ class ReplicaConfig: pd_p2p_comm_dtype: str = field( default='float16', metadata={ - "help": "Data type for PD disaggregation P2P communication.", - "choices": ['fp8', 'float16', 'float32', 'float64', 'bfloat16', 'int8', 'int16', 'int32', 'int64'], + "help": "Data type for PD disaggregation P2P communication. " + "Tested: fp8, float16, float32. " + "Other dtypes (float64, bfloat16, int8, int16, int32, int64) are " + "theoretically supported but untested.", + "choices": ['fp8', 'float16', 'float32'], }, ) diff --git a/vidur-alibabacloud/vidur/config/device_sku_config.py b/vidur-alibabacloud/vidur/config/device_sku_config.py index b4705367..f6204f87 100644 --- a/vidur-alibabacloud/vidur/config/device_sku_config.py +++ b/vidur-alibabacloud/vidur/config/device_sku_config.py @@ -63,25 +63,27 @@ class H800DeviceSKUConfig(BaseDeviceSKUConfig): @staticmethod def get_type(): return DeviceSKUType.H800 - -@dataclass -class H200DeviceSKUConfig(BaseDeviceSKUConfig): - fp16_tflops: int = 989 - fp8_tflops: int = 1979 - total_memory_gb: int = 141 - @staticmethod - def get_type(): - return DeviceSKUType.H200 +# NOTE: untested, for reference only (未经测试，仅供参考) +# @dataclass +# class H200DeviceSKUConfig(BaseDeviceSKUConfig): +# fp16_tflops: int = 989 +# fp8_tflops: int = 1979 +# total_memory_gb: int = 141 +# +# @staticmethod +# def get_type(): +# return DeviceSKUType.H200 # GB200 NVL72 -@dataclass -class GB200DeviceSKUConfig(BaseDeviceSKUConfig): - fp16_tflops: int = 2500 - fp8_tflops: int = 5000 - total_memory_gb: int = 192 - - @staticmethod - def get_type(): - return DeviceSKUType.GB200 \ No newline at end of file +# NOTE: untested, for reference only (未经测试，仅供参考) +# @dataclass +# class GB200DeviceSKUConfig(BaseDeviceSKUConfig): +# fp16_tflops: int = 2500 +# fp8_tflops: int = 5000 +# total_memory_gb: int = 192 +# +# @staticmethod +# def get_type(): +# return DeviceSKUType.GB200 \ No newline at end of file diff --git a/vidur-alibabacloud/vidur/execution_time_predictor/sklearn_execution_time_predictor.py b/vidur-alibabacloud/vidur/execution_time_predictor/sklearn_execution_time_predictor.py index a3e505af..d54880bd 100644 --- a/vidur-alibabacloud/vidur/execution_time_predictor/sklearn_execution_time_predictor.py +++ b/vidur-alibabacloud/vidur/execution_time_predictor/sklearn_execution_time_predictor.py @@ -681,6 +681,14 @@ def _train_models(self) -> Dict[str, BaseEstimator]: return models def _predict_for_compute_models_by_aicb(self) -> Dict[str, Any]: + # NOTE: AICB backend uses empirical linear formulas (time = a * batch_size + b) + # instead of ML model predictions. These formulas are derived from profiling data + # on specific hardware (H20) and represent the designed approach for AICB backend, + # not a temporary placeholder. + # 注意: AICB 后端使用基于经验的线性公式 (time = a * batch_size + b) + # 代替 ML 模型预测。这些公式来自特定硬件 (H20) 的 profiling 数据， + # 是 AICB 后端的设计方案，而非临时占位代码。 + # Store prediction results / 存储预测结果 predictions = {} @@ -1291,7 +1299,7 @@ def _get_attention_prefill_execution_time(self, batch: Batch) -> float: return 0 kv_cache_sizes, prefill_chunk_sizes = zip(*prefill_params) - print(len(prefill_chunk_sizes)) + logger.debug(f"prefill_chunk_sizes length: {len(prefill_chunk_sizes)}") agg_kv_cache_size = sum(kv_cache_sizes) agg_prefill_chunk_size = sum([x**2 for x in prefill_chunk_sizes]) ** 0.5 diff --git a/vidur-alibabacloud/vidur/metrics/metrics_store.py b/vidur-alibabacloud/vidur/metrics/metrics_store.py index 601d038e..e5247395 100644 --- a/vidur-alibabacloud/vidur/metrics/metrics_store.py +++ b/vidur-alibabacloud/vidur/metrics/metrics_store.py @@ -300,8 +300,7 @@ def _store_bar_plot( y=list(data.values()), labels={"x": x_label, "y": y_label}, ) - fig.write_image(f"{base_path}/{plot_name}.png") - _safe_write_image(fig, f"{base_path}/{plot_name}.png") # qoder + _safe_write_image(fig, f"{base_path}/{plot_name}.png") def _store_operation_metrics(self, base_plot_path: str): if not self._config.store_operation_metrics: diff --git a/vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py b/vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py index 312e5a38..e8cd5b73 100644 --- a/vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py +++ b/vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py @@ -73,9 +73,20 @@ def get_mla_kvcache_size(self, config, use_fp8): # TODO(tianhao909): re-verify KV cache calc for DeepSeek/Qwen3 # TODO(tianhao909): 重新核实 DeepSeek/Qwen3 的 KV Cache 计算方法 - # TODO(tianhao909): use per-request prefill/decode seq_len for PD separation - # TODO(tianhao909): PD 分离时传入不同 req 的 prefill_input_seq_len 和 decode_output_seq_len - # TODO(tianhao909): optimize from static to dynamic token allocation + # TODO(tianhao909): use per-request prefill/decode seq_len for PD separation. + # Currently prefill_input_seq_len and decode_output_seq_len are hardcoded to 1, + # which significantly underestimates KV cache memory per request. + # In production, a single request may have thousands of tokens. + # Fixing this requires passing actual seq_len from the request generator or + # using config-level max_prefill_tokens / max_decode_tokens. + # This is an architectural change that affects the entire memory planning pipeline. + # TODO(tianhao909): PD 分离时应传入不同 req 的实际 prefill_input_seq_len 和 decode_output_seq_len。 + # 当前硬编码 seq_len=1 会严重低估每请求的 KV cache 内存。 + # 实际场景中单个请求可能有数千个 token。 + # 修复此问题需要从请求生成器传入实际 seq_len，或使用配置级的 max_prefill_tokens / max_decode_tokens。 + # 这是一个影响整个内存规划流程的架构变更。 + # TODO(tianhao909): optimize from static to dynamic token allocation. + # Currently allocates by max_request_tokens statically, could be optimized to dynamic allocation. # TODO(tianhao909): 目前按 max_request_tokens 静态分配，后续可优化为动态分配 def _get_kv_cache_memory_per_layer_per_request(self) -> int: """Calculate KV cache memory per layer per request (bytes). @@ -84,14 +95,14 @@ def _get_kv_cache_memory_per_layer_per_request(self) -> int: # 当前仅 DeepSeek-671B 使用 MLA KV Cache if self._replica_config.model_name in ['deepseek-671B']: kvcache_size_all_layers_per_token = self.get_mla_kvcache_size(self._replica_config.model_config, self.use_fp8) - kvcache_size_per_layer_per_token = kvcache_size_all_layers_per_token / self._replica_config.model_config.num_hidden_layers + kvcache_size_per_layer_per_token = kvcache_size_all_layers_per_token // self._replica_config.model_config.num_hidden_layers prefill_input_seq_len = 1 decode_output_seq_len = 1 kvcache_size_per_layer_per_req = kvcache_size_per_layer_per_token * (prefill_input_seq_len + decode_output_seq_len) return kvcache_size_per_layer_per_req elif self._replica_config.model_name in ['qwen3-moe-235B', 'qwen3-next-80B']: kvcache_size_all_layers_per_token = self.get_mha_kvcache_size(self._replica_config.model_config, self.use_fp8) - kvcache_size_per_layer_per_token = kvcache_size_all_layers_per_token / self._replica_config.model_config.num_hidden_layers + kvcache_size_per_layer_per_token = kvcache_size_all_layers_per_token // self._replica_config.model_config.num_hidden_layers prefill_input_seq_len = 1 decode_output_seq_len = 1 kvcache_size_per_layer_per_req = kvcache_size_per_layer_per_token * (prefill_input_seq_len + decode_output_seq_len) @@ -229,7 +240,14 @@ def get_max_batch_size(self) -> int: ) # Return prefill max requests as system upper bound + # In PD disaggregation, prefill and decode clusters are scheduled independently, + # each with its own capacity. We return prefill's max requests here. + # Note: decode_num_requests may be smaller, but since the two clusters + # operate independently, each cluster uses its own limit for scheduling. # 返回Prefill的最大请求数 (作为系统上限) + # PD分离下，prefill 和 decode 集群独立调度，各自有各自的容量上限。 + # 注意：decode_num_requests 可能更小，但由于两个集群独立运行， + # 各集群调度时使用各自的上限。 return int(prefill_num_requests) else: parameter_memory_per_device = self._get_parameter_memory_per_device() diff --git a/vidur-alibabacloud/vidur/simulator.py b/vidur-alibabacloud/vidur/simulator.py index 4b5aede2..ed3bd963 100644 --- a/vidur-alibabacloud/vidur/simulator.py +++ b/vidur-alibabacloud/vidur/simulator.py @@ -115,7 +115,7 @@ def run(self) -> None: _GLOBAL_AICB_CACHE.print_stats() _GLOBAL_AICB_CACHE.save_lookup_table() except Exception as e: - print(f"[WARNING] Cannot print AICB cache stats (无法打印AICB缓存统计): {e}") + logger.warning(f"Cannot print AICB cache stats (无法打印AICB缓存统计): {e}") def _write_output(self) -> None: logger.info("Writing output") diff --git a/vidur-alibabacloud/vidur/types/device_sku_type.py b/vidur-alibabacloud/vidur/types/device_sku_type.py index 70696e1e..a7c8446f 100644 --- a/vidur-alibabacloud/vidur/types/device_sku_type.py +++ b/vidur-alibabacloud/vidur/types/device_sku_type.py @@ -7,5 +7,6 @@ class DeviceSKUType(BaseIntEnum): H100 = 3 H800 = 4 H20 = 5 - H200 = 6 - GB200 = 7 + # NOTE: untested, for reference only (未经测试，仅供参考) + # H200 = 6 + # GB200 = 7 diff --git a/vidur-alibabacloud/vidur/utils/mfu_calculator.py b/vidur-alibabacloud/vidur/utils/mfu_calculator.py index 8f3d8b13..81c613da 100644 --- a/vidur-alibabacloud/vidur/utils/mfu_calculator.py +++ b/vidur-alibabacloud/vidur/utils/mfu_calculator.py @@ -1,8 +1,11 @@ from vidur.config import ReplicaConfig from vidur.entities import BatchStage from vidur.entities.request import RequestType +from vidur.logger import init_logger from vidur.utils.param_counter import ParamCounter +logger = init_logger(__name__) + # MoE模型列表：这些模型需要区分 prefill/decode 的参数量 # MoE model list: These models need separate prefill/decode parameter counts @@ -39,11 +42,16 @@ def __init__(self, replica_config: ReplicaConfig): self._decode_num_params_per_device = params_result[2] # Decode参数量 | Decode params # 打印重要信息便于检查 | Print important info for verification - print(f"[MFUCalculator] MoE model PD separation mode (MoE模型 PD分离模式)") - print(f"[MFUCalculator] model_name={self._model_name}") - print(f"[MFUCalculator] num_params_per_device (total)={self._num_params_per_device / 1024 / 1024 / 1024:.4f} GB") - print(f"[MFUCalculator] prefill_num_params_per_device={self._prefill_num_params_per_device / 1024 / 1024 / 1024:.4f} GB") - print(f"[MFUCalculator] decode_num_params_per_device={self._decode_num_params_per_device / 1024 / 1024 / 1024:.4f} GB") + # TODO(tianhao909): ParamCounter returns memory bytes (not param count) for new models, + # causing MFU semantic inconsistency (2*tokens*bytes instead of 2*tokens*params). + # This is a known limitation; MFU values for MoE models are approximate. + # TODO(tianhao909): 新模型的 ParamCounter 返回内存字节数而非参数个数， + # 导致 MFU 语义不一致（2*tokens*bytes 而非 2*tokens*params），MoE 模型的 MFU 为近似值。 + logger.debug(f"[MFUCalculator] MoE model PD separation mode (MoE模型 PD分离模式)") + logger.debug(f"[MFUCalculator] model_name={self._model_name}") + logger.debug(f"[MFUCalculator] num_params_per_device (total)={self._num_params_per_device / 1024 / 1024 / 1024:.4f} GB") + logger.debug(f"[MFUCalculator] prefill_num_params_per_device={self._prefill_num_params_per_device / 1024 / 1024 / 1024:.4f} GB") + logger.debug(f"[MFUCalculator] decode_num_params_per_device={self._decode_num_params_per_device / 1024 / 1024 / 1024:.4f} GB") else: # 普通模型：返回单个值 # Normal model: returns single value @@ -51,9 +59,9 @@ def __init__(self, replica_config: ReplicaConfig): self._prefill_num_params_per_device = self._num_params_per_device self._decode_num_params_per_device = self._num_params_per_device - print(f"[MFUCalculator] Normal model mode (普通模型模式)") - print(f"[MFUCalculator] model_name={self._model_name}") - print(f"[MFUCalculator] num_params_per_device={self._num_params_per_device}") + logger.debug(f"[MFUCalculator] Normal model mode (普通模型模式)") + logger.debug(f"[MFUCalculator] model_name={self._model_name}") + logger.debug(f"[MFUCalculator] num_params_per_device={self._num_params_per_device}") model_config = replica_config.model_config @@ -128,7 +136,7 @@ def get_mfu(self, batch_stage: BatchStage) -> float: # 防止除零错误：如果execution_time为0，返回0 # Prevent division by zero: return 0 if execution_time is 0 if batch_stage.execution_time == 0: - print(f"[WARNING] batch_stage.execution_time is 0, returning MFU as 0") + logger.warning(f"batch_stage.execution_time is 0, returning MFU as 0") return 0.0 total_flops_per_second = total_flops / batch_stage.execution_time From bf7f98986165c882f7414b8a51cdf4b426470781 Mon Sep 17 00:00:00 2001 From: tianhao909 <843101550@qq.com> Date: Tue, 14 Apr 2026 11:00:14 +0000 Subject: [PATCH 3/8] test(vidur): add run_scenarios.sh 4-scenario test results to report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [EN] - Ran all 4 scenarios from run_scenarios.sh (Qwen3-Next-80B no-PD, Qwen3-Next-80B PD, DeepSeek-671B PD, Qwen3-MoE-235B PD) - All 4 scenarios passed on H20 GPU with FP8 + AICB backend - Updated test_tutorial_report_cn.md with detailed results table [ZH] - 运行 run_scenarios.sh 全部 4 个场景 (Qwen3-Next-80B 无PD, Qwen3-Next-80B PD, DeepSeek-671B PD, Qwen3-MoE-235B PD) - 4/4 场景在 H20 GPU + FP8 + AICB 后端下全部通过 - 更新 test_tutorial_report_cn.md 补充四场景测试结果表格 --- .../tests/test_tutorial_report_cn.md | 48 +++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/vidur-alibabacloud/tests/test_tutorial_report_cn.md b/vidur-alibabacloud/tests/test_tutorial_report_cn.md index cbfdb825..62334a31 100644 --- a/vidur-alibabacloud/tests/test_tutorial_report_cn.md +++ b/vidur-alibabacloud/tests/test_tutorial_report_cn.md @@ -181,20 +181,62 @@ DeepSeek-671B 模型参数量极大, 在 H20 (141GB) 上需要较高的并行度 --- -## 5. 测试结果汇总 +## 5. 集成测试 — run_scenarios.sh 四场景套件 + +### 5.1 场景概览 + +| 场景 | 模型 | PD 分离 | 集群配置 | 调度策略 | +|:---:|------|---------|---------|---------| +| 1 | Qwen3-Next-80B | 否 (ratio=1) | 32 replicas, tp=1, pp=1, ep=32 | lor + sarathi | +| 2 | Qwen3-Next-80B | 是 (P=2, D=6) | 8 replicas, tp=1, pp=1 | split_wise | +| 3 | DeepSeek-671B | 是 (P=2, D=6) | 8 replicas, tp=8, pp=1, ep=8 | split_wise | +| 4 | Qwen3-MoE-235B | 是 (P=2, D=6) | 8 replicas, tp=4, pp=1, ep=4 | split_wise | + +**公共参数**: H20 GPU, FP8, AICB 后端, Poisson QPS=100, 4 请求, prefill=100 tokens, decode=8 tokens + +### 5.2 运行命令 + +```bash +# 通过脚本运行全部四场景 (需要 vidur conda 环境) +bash examples/vidur-ali-scenarios/run_scenarios.sh --all + +# 或单独运行某个场景 +bash examples/vidur-ali-scenarios/run_scenarios.sh --scenario 1 +``` + +### 5.3 测试结果 + +| 场景 | 模型 | 模拟结束时间 | 状态 | 说明 | +|:---:|------|------------|------|------| +| 1 | Qwen3-Next-80B (无PD) | 0.016s | **通过** | AICB CSV 缺失使用默认执行时间 (预期行为) | +| 2 | Qwen3-Next-80B (PD) | 0.016s | **通过** | PD 分离 P=2 D=6 正常调度 | +| 3 | DeepSeek-671B (PD) | 0.017s | **通过** | MoE 671B 参数 + MLA KV cache | +| 4 | Qwen3-MoE-235B (PD) | 0.016s | **通过** | MoE 235B 参数 + MHA KV cache | + +**结果: 4/4 场景全部通过** + +### 5.4 已知提示 + +- AICB 后端会输出 `AICB command failed` / `无法找到任何AICB CSV文件` 错误 → 这是**预期行为**, 表示无实际 profiling 数据, 系统自动使用经验线性公式估算执行时间 +- numpy `RuntimeWarning: invalid value encountered in subtract` → 统计指标计算中的边界情况, 不影响仿真结果 +- `run_scenarios.sh` 脚本要求 `conda activate vidur` 环境; 也可直接用 `python -m vidur.main` 在 base 环境中运行 + +--- + +## 6. 测试结果汇总 | 测试类型 | 测试项 | 状态 | 说明 | |---------|--------|------|------| | Layer 1 | PD 分离单元测试 (10 cases) | **全部通过** | 无外部依赖 | | Layer 2 | Llama-3-8B Native Vidur | **通过** | 需要 processed_traces + profiling 数据 | | Layer 2 | DeepSeek-671B AICB (H20 FP8) | **通过** | 需要 H20 + TP=8 + FP8 配置 | -| Layer 3 | run_scenarios.sh 四场景 | **未测试** | 需要 AICB Docker 环境 | +| Layer 2 | run_scenarios.sh 四场景套件 | **全部通过** | 4/4 场景通过 (AICB 后端 + H20 FP8) | | Layer 3 | Llama-3-8B SimAI Simulation | **未测试** | 需要编译 SimAI ns3 | | Layer 3 | Llama-3-8B SimAI Analytical | **未测试** | 需要编译 SimAI Analytical | --- -## 6. 已知限制与依赖 +## 7. 已知限制与依赖 | 限制 | 说明 | |------|------| From 1b547ca497ea93523256ee5c46b6a161a019a639 Mon Sep 17 00:00:00 2001 From: tianhao909 <843101550@qq.com> Date: Wed, 15 Apr 2026 13:56:51 +0000 Subject: [PATCH 4/8] fix(PR#268): correct FlashMLA h_q alignment issue description + add verification appendix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Correct inaccurate "bs>1 fails" wording in config.py and execution_time.py to precise root cause: FlashMLA flash_mla_sparse_fwd kernel h_q alignment (B_H=64) - Add Appendix A to test_tutorial_report_cn.md with: - Full AICB tp=1/2/4/8 prefill + tp=8 decode verification matrix - FlashMLA source code evidence and call stack - Vidur degradation path validation - Environment info 修正 config.py 和 execution_time.py 中不准确的 "bs>1 失败" 描述， 改为精确的根因：FlashMLA flash_mla_sparse_fwd kernel 的 h_q 对齐要求 (B_H=64)。 在 test_tutorial_report_cn.md 中新增附录 A，包含完整验证矩阵、 FlashMLA 源码依据、调用栈、Vidur 降级验证和环境信息。 --- vidur-alibabacloud/README-vidur.md | 18 +-- vidur-alibabacloud/README.md | 23 ++- vidur-alibabacloud/README_CN.md | 23 ++- .../vidur-ali-scenarios/run_scenarios.sh | 18 +-- .../tests/test_tutorial_report_cn.md | 142 +++++++++++++++++- vidur-alibabacloud/vidur/config/config.py | 50 +++--- .../vidur/config/device_sku_config.py | 26 ---- .../vidur/config/node_sku_config.py | 9 ++ .../vidur/entities/execution_time.py | 24 +-- .../vidur/events/global_schedule_event.py | 3 - .../vidur/metrics/metrics_store.py | 4 - .../global_scheduler/lor_global_scheduler.py | 3 - .../sarathi_replica_scheduler.py | 1 - .../vidur/utils/base_registry.py | 1 - 14 files changed, 217 insertions(+), 128 deletions(-) diff --git a/vidur-alibabacloud/README-vidur.md b/vidur-alibabacloud/README-vidur.md index 00c08c04..786ea177 100644 --- a/vidur-alibabacloud/README-vidur.md +++ b/vidur-alibabacloud/README-vidur.md @@ -180,10 +180,10 @@ Any use of third-party trademarks or logos are subject to those third-party's po | 场景 | 模型 | ws | TP | PP | EP | PD分离 | 调度器 | |------|------|----|----|----|-----|--------|--------| -| 1 | Qwen3-Next-80B | 32 | 1 | 1 | 32 | 否 | lor | -| 2 | Qwen3-Next-80B | 8 (P=2,D=6) | 1 | 1 | auto | 是 | split_wise | -| 3 | DeepSeek-671B | 8 (P=2,D=6) | 8 | 1 | 8 | 是 | split_wise | -| 4 | Qwen3-MoE-235B | 8 (P=2,D=6) | 4 | 1 | 4 | 是 | split_wise | +| 1 | Qwen3-Next-80B | 32 | 1 | 1 | auto (=ws) | 否 | lor | +| 2 | Qwen3-Next-80B | 8 (P=2,D=6) | 1 | 1 | auto (=ws) | 是 | split_wise | +| 3 | DeepSeek-671B | 8 (P=2,D=6) | 8 | 1 | auto (=ws) | 是 | split_wise | +| 4 | Qwen3-MoE-235B | 8 (P=2,D=6) | 4 | 1 | auto (=ws) | 是 | split_wise | ### 使用方法 @@ -266,7 +266,7 @@ python -m vidur.main \ --replica_config_decode_num_pipeline_stages 1 ``` -**场景 3: DeepSeek-671B PD分离 (tp=8, ep=8, split_wise)** +**场景 3: DeepSeek-671B PD分离 (tp=8, EP=auto, split_wise)** ```sh python -m vidur.main \ @@ -292,11 +292,10 @@ python -m vidur.main \ --replica_scheduler_config_type split_wise \ --replica_config_model_name deepseek-671B \ --replica_config_tensor_parallel_size 8 \ - --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 8 + --replica_config_num_pipeline_stages 1 ``` -**场景 4: Qwen3-MoE-235B PD分离 (tp=4, ep=4, split_wise)** +**场景 4: Qwen3-MoE-235B PD分离 (tp=4, EP=auto, split_wise)** ```sh python -m vidur.main \ @@ -322,6 +321,5 @@ python -m vidur.main \ --replica_scheduler_config_type split_wise \ --replica_config_model_name qwen3-moe-235B \ --replica_config_tensor_parallel_size 4 \ - --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 4 + --replica_config_num_pipeline_stages 1 ``` \ No newline at end of file diff --git a/vidur-alibabacloud/README.md b/vidur-alibabacloud/README.md index 7af82c86..df99cfcd 100644 --- a/vidur-alibabacloud/README.md +++ b/vidur-alibabacloud/README.md @@ -37,7 +37,7 @@ Vidur ([original](https://github.com/microsoft/vidur)) is a simulation framework - **Data Parallel (DP)** - **Tensor Parallel (TP)** - **Pipeline Parallel (PP)** - - **Expert Parallel (EP)** (support in progress) + - **Expert Parallel (EP)** (auto-set to cluster world_size, manual override not supported) Works for both **dense** and **Mixture-of-Experts (MoE)** models (MoE support in progress). - **Multiple Execution-Time Prediction Backends** — Choose from: @@ -88,8 +88,8 @@ We gratefully acknowledge these resources for providing the foundational analysi ## Supported Models -- **DeepSeek-V3-671B** (SimAI PP/EP communication and GPU memory allocation module adaptations in progress) -- **Qwen3-MoE-235B**, **Qwen3-Next-80B** (SimAI PP/EP communication and GPU memory allocation module adaptations in progress) +- **DeepSeek-V3-671B** (SimAI PP communication module in progress; EP auto-set to world_size; GPU memory management supported) +- **Qwen3-MoE-235B**, **Qwen3-Next-80B** (SimAI PP communication module in progress; EP auto-set to world_size; GPU memory management supported) - **meta-llama/Meta-Llama-3-8B** / **Meta-Llama-3-70B** - **meta-llama/Llama-2-7b-hf** / **Llama-2-70b-hf** - **codellama/CodeLlama-34b-Instruct-hf** @@ -187,7 +187,6 @@ python -m vidur.main --replica_config_pd_p2p_comm_bandwidth 800 \ --replica_config_model_name deepseek-671B \ --replica_config_tensor_parallel_size 2 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 8 \ --random_forrest_execution_time_predictor_config_backend aicb ``` @@ -214,7 +213,6 @@ python -m vidur.main \ --replica_config_model_name deepseek-671B \ --replica_config_tensor_parallel_size 2 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 8 \ --random_forrest_execution_time_predictor_config_backend aicb ``` @@ -252,7 +250,6 @@ python -m vidur.main \ --replica_config_model_name meta-llama/Meta-Llama-3-8B \ --replica_config_tensor_parallel_size 4 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 1 \ --random_forrest_execution_time_predictor_config_backend simai_simulation \ --random_forrest_execution_time_predictor_config_simai_dir ../ \ --random_forrest_execution_time_predictor_config_simai_simulation_topo ../Spectrum-X_128g_8gps_100Gbps_A100 \ @@ -287,7 +284,6 @@ python -m vidur.main \ --replica_config_model_name meta-llama/Meta-Llama-3-8B \ --replica_config_tensor_parallel_size 4 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 1 \ --random_forrest_execution_time_predictor_config_backend simai_analytical ``` @@ -314,7 +310,6 @@ python -m vidur.main \ --replica_config_model_name meta-llama/Meta-Llama-3-8B \ --replica_config_tensor_parallel_size 4 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 1 \ --random_forrest_execution_time_predictor_config_backend vidur ``` @@ -339,12 +334,12 @@ The following scenarios are pre-configured in `run_scenarios.sh`. All scenarios | Scenario | Model | PD Separation | World Size | TP | PP | EP | Global Scheduler | |----------|-------|---------------|------------|----|----|------------|------------------| -| 1 | Qwen3-Next-80B (MoE) | No | 32 (dp=32) | 1 | 1 | 1 (default) | lor | -| 2 | Qwen3-Next-80B (MoE) | Yes (P=2, D=6) | 8 | 1 | 1 | 1 (default) | split_wise | -| 3 | DeepSeek-671B (MoE) | Yes (P=2, D=6) | 8 | 8 | 1 | 8 | split_wise | -| 4 | Qwen3-MoE-235B (MoE) | Yes (P=2, D=6) | 8 | 4 | 1 | 4 | split_wise | +| 1 | Qwen3-Next-80B (MoE) | No | 32 (dp=32) | 1 | 1 | auto (=world_size) | lor | +| 2 | Qwen3-Next-80B (MoE) | Yes (P=2, D=6) | 8 | 1 | 1 | auto (=world_size) | split_wise | +| 3 | DeepSeek-671B (MoE) | Yes (P=2, D=6) | 8 | 8 | 1 | auto (=world_size) | split_wise | +| 4 | Qwen3-MoE-235B (MoE) | Yes (P=2, D=6) | 8 | 4 | 1 | auto (=world_size) | split_wise | -> **Note:** All four models use Mixture-of-Experts (MoE) architecture. The EP column reflects the explicit `--replica_config_expert_model_parallel_size` value set in the script; scenarios without an explicit EP setting use the default value of 1. +> **Note:** All four models use Mixture-of-Experts (MoE) architecture. EP is automatically set to the cluster world_size at runtime and cannot be manually overridden. #### Output Files @@ -396,7 +391,7 @@ Each run produces the following directory: | `--replica_config_model_name` | meta-llama/Llama-2-7b-hf | Model name (DeepSeek-671B, Qwen3-MoE-235B, Qwen3-Next-80B, etc.) | | `--replica_config_tensor_parallel_size` | 1 | Tensor parallelism size (TP) | | `--replica_config_num_pipeline_stages` | 1 | Number of pipeline stages (PP) | -| `--replica_config_expert_model_parallel_size` | 1 | Expert model parallelism size (EP) | +| `--replica_config_expert_model_parallel_size` | 1 | Expert model parallelism size (EP) — auto-set to world_size internally. Passing a value != world_size raises ValueError. Manual override not recommended. | | `--random_forrest_execution_time_predictor_config_backend` | vidur | Backend for execution time prediction (`vidur`, `simai_simulation`, `simai_analytical`, `aicb`, etc.). **Note:** `simai_simulation` and `simai_analytical` currently only model TP communication and do not support pipeline or expert parallelism. | | `--random_forrest_execution_time_predictor_config_simai_dir` | `../` | Root directory of the SimAI simulator (only effective when backend = `simai_simulation`) | | `--random_forrest_execution_time_predictor_config_simai_simulation_topo` | `../example/topo` | Path to SimAI topology file (only effective when backend = `simai_simulation`) | diff --git a/vidur-alibabacloud/README_CN.md b/vidur-alibabacloud/README_CN.md index e37cc9ba..874a794f 100644 --- a/vidur-alibabacloud/README_CN.md +++ b/vidur-alibabacloud/README_CN.md @@ -36,7 +36,7 @@ Vidur（[原版](https://github.com/microsoft/vidur)）是一个大语言模型 - **数据并行（DP）** - **张量并行（TP）** - **流水线并行（PP）** - - **专家并行（EP）**（适配中） + - **专家并行（EP）**（自动设为 cluster world_size，不支持手动指定） 同时支持 **Dense** 模型和 **混合专家（MoE）** 模型（MoE 适配中）。 - **多种执行时间预测后端** — 可选： @@ -87,8 +87,8 @@ Vidur（[原版](https://github.com/microsoft/vidur)）是一个大语言模型 ## 支持的模型 -- **DeepSeek-V3-671B**（SimAI PP/EP 通信及 GPU 显存管理模块适配中） -- **Qwen3-MoE-235B**、**Qwen3-Next-80B**（SimAI PP/EP 通信及 GPU 显存管理模块适配中） +- **DeepSeek-V3-671B**（SimAI PP 通信模块适配中；EP 自动设为 world_size；GPU 显存管理已支持） +- **Qwen3-MoE-235B**、**Qwen3-Next-80B**（SimAI PP 通信模块适配中；EP 自动设为 world_size；GPU 显存管理已支持） - **meta-llama/Meta-Llama-3-8B** / **Meta-Llama-3-70B** - **meta-llama/Llama-2-7b-hf** / **Llama-2-70b-hf** - **codellama/CodeLlama-34b-Instruct-hf** @@ -186,7 +186,6 @@ python -m vidur.main --replica_config_pd_p2p_comm_bandwidth 800 \ --replica_config_model_name deepseek-671B \ --replica_config_tensor_parallel_size 2 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 8 \ --random_forrest_execution_time_predictor_config_backend aicb ``` @@ -213,7 +212,6 @@ python -m vidur.main \ --replica_config_model_name deepseek-671B \ --replica_config_tensor_parallel_size 2 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 8 \ --random_forrest_execution_time_predictor_config_backend aicb ``` @@ -251,7 +249,6 @@ python -m vidur.main \ --replica_config_model_name meta-llama/Meta-Llama-3-8B \ --replica_config_tensor_parallel_size 4 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 1 \ --random_forrest_execution_time_predictor_config_backend simai_simulation \ --random_forrest_execution_time_predictor_config_simai_dir ../ \ --random_forrest_execution_time_predictor_config_simai_simulation_topo ../Spectrum-X_128g_8gps_100Gbps_A100 \ @@ -286,7 +283,6 @@ python -m vidur.main \ --replica_config_model_name meta-llama/Meta-Llama-3-8B \ --replica_config_tensor_parallel_size 4 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 1 \ --random_forrest_execution_time_predictor_config_backend simai_analytical ``` @@ -313,7 +309,6 @@ python -m vidur.main \ --replica_config_model_name meta-llama/Meta-Llama-3-8B \ --replica_config_tensor_parallel_size 4 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 1 \ --random_forrest_execution_time_predictor_config_backend vidur ``` @@ -338,12 +333,12 @@ bash examples/vidur-ali-scenarios/run_scenarios.sh --all | 场景 | 模型 | PD 分离 | World Size | TP | PP | EP | 全局调度器 | |------|------|---------|------------|----|----|------------|------------| -| 1 | Qwen3-Next-80B (MoE) | 无 | 32 (dp=32) | 1 | 1 | 1（默认） | lor | -| 2 | Qwen3-Next-80B (MoE) | 是（P=2, D=6） | 8 | 1 | 1 | 1（默认） | split_wise | -| 3 | DeepSeek-671B (MoE) | 是（P=2, D=6） | 8 | 8 | 1 | 8 | split_wise | -| 4 | Qwen3-MoE-235B (MoE) | 是（P=2, D=6） | 8 | 4 | 1 | 4 | split_wise | +| 1 | Qwen3-Next-80B (MoE) | 无 | 32 (dp=32) | 1 | 1 | auto (=world_size) | lor | +| 2 | Qwen3-Next-80B (MoE) | 是（P=2, D=6） | 8 | 1 | 1 | auto (=world_size) | split_wise | +| 3 | DeepSeek-671B (MoE) | 是（P=2, D=6） | 8 | 8 | 1 | auto (=world_size) | split_wise | +| 4 | Qwen3-MoE-235B (MoE) | 是（P=2, D=6） | 8 | 4 | 1 | auto (=world_size) | split_wise | -> **说明：** 四个模型均使用混合专家（MoE）架构。EP 列反映脚本中 `--replica_config_expert_model_parallel_size` 的显式设定值；未显式指定时使用默认值 1。 +> **说明：** 四个模型均使用混合专家（MoE）架构。EP 在运行时自动设为 cluster world_size，不支持手动指定。 #### 输出文件说明 @@ -395,7 +390,7 @@ bash examples/vidur-ali-scenarios/run_scenarios.sh --all | `--replica_config_model_name` | meta-llama/Llama-2-7b-hf | 模型名称（DeepSeek-671B、Qwen3-MoE-235B、Qwen3-Next-80B 等） | | `--replica_config_tensor_parallel_size` | 1 | 张量并行大小（TP） | | `--replica_config_num_pipeline_stages` | 1 | 流水线阶段数（PP） | -| `--replica_config_expert_model_parallel_size` | 1 | 专家并行大小（EP） | +| `--replica_config_expert_model_parallel_size` | 1 | 专家并行大小（EP）— 内部自动设为 world_size，用户传入非 world_size 值将报错。不建议手动指定。 | | `--random_forrest_execution_time_predictor_config_backend` | vidur | 执行时间预测后端（`vidur`、`simai_simulation`、`simai_analytical`、`aicb` 等）。**注意：** `simai_simulation` 和 `simai_analytical` 当前仅建模 TP 通信，不支持流水线或专家并行。 | | `--random_forrest_execution_time_predictor_config_simai_dir` | `../` | SimAI 模拟器根目录（仅在 backend = `simai_simulation` 时生效） | | `--random_forrest_execution_time_predictor_config_simai_simulation_topo` | `../example/topo` | SimAI 拓扑文件路径（仅在 backend = `simai_simulation` 时生效） | diff --git a/vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh b/vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh index 82d00dc8..1bcc5a66 100644 --- a/vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh +++ b/vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh @@ -18,8 +18,8 @@ # 场景说明: # 1 Qwen3-Next-80B 无PD分离 ws=32 (dp=32, tp=1, pp=1, ep=32) 调度: lor # 2 Qwen3-Next-80B PD分离 ws=8 (P=2, D=6, tp=1, pp=1) 调度: split_wise -# 3 DeepSeek-671B PD分离 ws=8 (P=2, D=6, tp=8, pp=1, ep=8) 调度: split_wise -# 4 Qwen3-MoE-235B PD分离 ws=8 (P=2, D=6, tp=4, pp=1, ep=4) 调度: split_wise +# 3 DeepSeek-671B PD分离 ws=8 (P=2, D=6, tp=8, pp=1, EP auto) 调度: split_wise +# 4 Qwen3-MoE-235B PD分离 ws=8 (P=2, D=6, tp=4, pp=1, EP auto) 调度: split_wise # # 环境要求: # conda activate vidur @@ -218,14 +218,14 @@ run_scenario_2() { # ----------------------------------------------------------------------- # 场景 3: DeepSeek-671B PD分离 # 总 replica=8; pd_node_ratio=0.25 → prefill dp=2, decode dp=6 -# ws = tp(8) × pp(1) × dp = 16(P)/48(D)，ep = 8 +# ws = tp(8) × pp(1) × dp = 16(P)/48(D)，EP auto-set to world_size # 调度: global=split_wise, replica=split_wise # ----------------------------------------------------------------------- run_scenario_3() { local ts ts="$(date +%Y%m%d_%H%M%S)" local log_file="$LOG_DIR/scenario_3_${ts}.log" - echo "[INFO] === Scenario 3: DeepSeek-671B, PD, tp=8, ep=8, split_wise (场景3: PD分离, tp=8, ep=8) ===" + echo "[INFO] === Scenario 3: DeepSeek-671B, PD, tp=8, EP=auto, split_wise (场景3: PD分离, tp=8, EP=auto) ===" echo "[INFO] Log (日志): $log_file" cd "$VIDUR_ROOT" set +o pipefail @@ -238,7 +238,6 @@ run_scenario_3() { --replica_config_model_name deepseek-671B \ --replica_config_tensor_parallel_size 8 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 8 \ 2>&1 | tee "$log_file" local exit_code=${PIPESTATUS[0]} set -o pipefail @@ -253,14 +252,14 @@ run_scenario_3() { # ----------------------------------------------------------------------- # 场景 4: Qwen3-MoE-235B PD分离 # 总 replica=8; pd_node_ratio=0.25 → prefill dp=2, decode dp=6 -# ws = tp(4) × pp(1) × dp = 8(P)/24(D)，ep = 4 +# ws = tp(4) × pp(1) × dp = 8(P)/24(D)，EP auto-set to world_size # 调度: global=split_wise, replica=split_wise # ----------------------------------------------------------------------- run_scenario_4() { local ts ts="$(date +%Y%m%d_%H%M%S)" local log_file="$LOG_DIR/scenario_4_${ts}.log" - echo "[INFO] === Scenario 4: Qwen3-MoE-235B, PD, tp=4, ep=4, split_wise (场景4: PD分离, tp=4, ep=4) ===" + echo "[INFO] === Scenario 4: Qwen3-MoE-235B, PD, tp=4, EP=auto, split_wise (场景4: PD分离, tp=4, EP=auto) ===" echo "[INFO] Log (日志): $log_file" cd "$VIDUR_ROOT" set +o pipefail @@ -273,7 +272,6 @@ run_scenario_4() { --replica_config_model_name qwen3-moe-235B \ --replica_config_tensor_parallel_size 4 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 4 \ 2>&1 | tee "$log_file" local exit_code=${PIPESTATUS[0]} set -o pipefail @@ -297,8 +295,8 @@ Usage (用法): Scenarios (场景列表): 1 Qwen3-Next-80B no PD (无PD分离) ws=32 scheduler: lor 2 Qwen3-Next-80B PD (PD分离) ws=8 (P=2,D=6) scheduler: split_wise - 3 DeepSeek-671B PD (PD分离) tp=8, ep=8 scheduler: split_wise - 4 Qwen3-MoE-235B PD (PD分离) tp=4, ep=4 scheduler: split_wise + 3 DeepSeek-671B PD (PD分离) tp=8, EP=auto scheduler: split_wise + 4 Qwen3-MoE-235B PD (PD分离) tp=4, EP=auto scheduler: split_wise Output dir (输出目录): examples/vidur-ali-scenarios/simulator_output// Log dir (日志目录): examples/vidur-ali-scenarios/logs/scenario__.log diff --git a/vidur-alibabacloud/tests/test_tutorial_report_cn.md b/vidur-alibabacloud/tests/test_tutorial_report_cn.md index 62334a31..95fc7d1e 100644 --- a/vidur-alibabacloud/tests/test_tutorial_report_cn.md +++ b/vidur-alibabacloud/tests/test_tutorial_report_cn.md @@ -107,7 +107,6 @@ python -m vidur.main \ --replica_config_model_name meta-llama/Meta-Llama-3-8B \ --replica_config_tensor_parallel_size 4 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 1 \ --random_forrest_execution_time_predictor_config_backend vidur ``` @@ -153,7 +152,6 @@ python -m vidur.main \ --replica_config_model_name deepseek-671B \ --replica_config_tensor_parallel_size 8 \ --replica_config_num_pipeline_stages 1 \ - --replica_config_expert_model_parallel_size 8 \ --random_forrest_execution_time_predictor_config_backend aicb \ --replica_config_device h20 ``` @@ -167,7 +165,7 @@ python -m vidur.main \ | 处理请求数 | 5 | | 集群配置 | 4 replicas, PD ratio=0.5 (2P+2D), H20 GPU | | 已知警告 | "AICB data is empty, using default execution time" — 预期行为 | -| 注意事项 | README 示例使用 TP=2 EP=8 在 H20 上会 OOM, 需要 TP=8 + FP8 才能跑通 | +| 注意事项 | EP 自动设为 world_size (=8)。DeepSeek-671B 在 H20 上需要 TP=8 + FP8 才能通过内存检查 | ### 4.3 GPU 内存分析 @@ -187,10 +185,10 @@ DeepSeek-671B 模型参数量极大, 在 H20 (141GB) 上需要较高的并行度 | 场景 | 模型 | PD 分离 | 集群配置 | 调度策略 | |:---:|------|---------|---------|---------| -| 1 | Qwen3-Next-80B | 否 (ratio=1) | 32 replicas, tp=1, pp=1, ep=32 | lor + sarathi | -| 2 | Qwen3-Next-80B | 是 (P=2, D=6) | 8 replicas, tp=1, pp=1 | split_wise | -| 3 | DeepSeek-671B | 是 (P=2, D=6) | 8 replicas, tp=8, pp=1, ep=8 | split_wise | -| 4 | Qwen3-MoE-235B | 是 (P=2, D=6) | 8 replicas, tp=4, pp=1, ep=4 | split_wise | +| 1 | Qwen3-Next-80B | 否 (ratio=1) | 32 replicas, tp=1, pp=1, EP=auto | lor + sarathi | +| 2 | Qwen3-Next-80B | 是 (P=2, D=6) | 8 replicas, tp=1, pp=1, EP=auto | split_wise | +| 3 | DeepSeek-671B | 是 (P=2, D=6) | 8 replicas, tp=8, pp=1, EP=auto | split_wise | +| 4 | Qwen3-MoE-235B | 是 (P=2, D=6) | 8 replicas, tp=4, pp=1, EP=auto | split_wise | **公共参数**: H20 GPU, FP8, AICB 后端, Poisson QPS=100, 4 请求, prefill=100 tokens, decode=8 tokens @@ -246,3 +244,133 @@ bash examples/vidur-ali-scenarios/run_scenarios.sh --scenario 1 | GPU 内存 | DeepSeek-671B 在 H20 上需要 TP>=8 + FP8 才能通过内存检查 | | SimAI 构建 | SimAI Simulation/Analytical 模式需要额外编译步骤 | | seq_len=1 | KV cache 计算中 seq_len 硬编码为 1 (已知限制, 见 TODO 注释) | + +--- + +## 附录 A: DeepSeek-V3-671B Prefill AICB Profiling 失败分析 + +> 测试日期: 2026-04-15 +> 测试目的: 定位并验证 AICB 对 DeepSeek-V3-671B prefill 做 GPU kernel profiling 时的失败根因 + +### A.1 问题描述 + +AICB 使用 `--aiob_enable` 对 DeepSeek-V3-671B prefill 做 GPU kernel profiling 时， +在 H20 (SM90) 上当 `tp>=4` 时运行崩溃，报错： + +``` +Assertion failed (/tmp/pip-req-build-gg88vm1n/csrc/sm90/prefill/sparse/fwd.cu:647): params.h_q % B_H == 0 +``` + +DeepSeek-V3 有 128 个 attention heads (`num_attention_heads=128`)， +AICB 按 `h_q = num_attention_heads / tp` 计算每个 TP rank 的 head 数量。 +当 tp>=4 时，h_q < 64，无法整除 FlashMLA 的 tile 常量 B_H=64。 + +### A.2 FlashMLA 官方依据 + +FlashMLA 的 SM90 prefill sparse kernel 在编译时定义 `B_H = 64`（对应 SM90 WGMMA 64×64 tile）， +运行时要求 `params.h_q % B_H == 0`，否则触发 CUDA 断言失败。 + +- 源码位置: [`csrc/sm90/prefill/sparse/config.h`](https://github.com/deepseek-ai/FlashMLA/blob/main/csrc/sm90/prefill/sparse/config.h) — `B_H = 64` 编译时常量 +- 断言位置: `csrc/sm90/prefill/sparse/fwd.cu:647` — `params.h_q % B_H == 0` +- FlashMLA 版本: `1.0.0+1408756` (pip install) + +### A.3 完整调用栈 + +``` +Vidur (vidur.main) + └── RandomForrestExecutionTimePredictor + └── ExecutionTimeSeries._generate_aicb_csv() + └── subprocess: python -m workload_generator.Vidur_workload_generator + └── AiobDeepSeek (prefill phase) + └── flash_mla.flash_mla_sparse_fwd() + └── SM90 CUDA kernel (fwd.cu:647) + └── ASSERTION FAILED: params.h_q % B_H == 0 +``` + +### A.4 完整验证矩阵 + +**Prefill 测试 (DeepSeek-V3-671B, seq=1024)** + +| tp | h_q=128/tp | bs=1 | bs=2 | bs=4 | bs=8 | 结论 | +|----|-----------|------|------|------|------|------| +| 1 | 128 | ✅ | ✅ | ✅ | ✅ | h_q=128, 128%64=0, 全部通过 | +| 2 | 64 | ✅ | ✅ | ✅ | ✅ | h_q=64, 64%64=0, 边界值通过 | +| 4 | 32 | ❌ | ❌ | ❌ | — | h_q=32, 32%64≠0, 全部失败 | +| 8 | 16 | ❌ | ❌ | ❌ | — | h_q=16, 16%64≠0, 全部失败 | + +**关键结论**: 失败完全由 tp 决定（h_q 对齐），与 bs 无关。tp=1/2 下 bs=2/4/8 全部通过， +tp=4/8 下即使 bs=1 也会失败。之前 "bs>1 fails" 的描述不准确。 + +**Decode 测试 (不受影响验证)** + +| 测试 | tp | bs | 预期 | 实际 | 说明 | +|------|-----|-----|------|------|------| +| tp=8 decode | 8 | 2 | PASS | **PASS** | decode 路径使用不同 kernel，不受 B_H 限制 | + +**测试命令示例:** + +```bash +# tp=1 prefill (PASS) +cd aicb && conda run -n vidur python -m workload_generator.Vidur_workload_generator \ + DeepSeek-671B ./scripts/inference_configs/deepseek_default.json \ + --seq_length 1024 --micro_batch 1 --world_size 1 \ + --tensor_model_parallel_size 1 --expert_model_parallel_size 1 \ + --aiob_enable --phase prefill + +# tp=4 prefill (FAIL — h_q=32, 32%64≠0) +cd aicb && conda run -n vidur python -m workload_generator.Vidur_workload_generator \ + DeepSeek-671B ./scripts/inference_configs/deepseek_default.json \ + --seq_length 1024 --micro_batch 1 --world_size 4 \ + --tensor_model_parallel_size 4 --expert_model_parallel_size 4 \ + --aiob_enable --phase prefill + +# tp=8 decode (PASS — decode uses different kernel) +cd aicb && conda run -n vidur python -m workload_generator.Vidur_workload_generator \ + DeepSeek-671B ./scripts/inference_configs/deepseek_default.json \ + --seq_length 1024 --micro_batch 2 --world_size 8 \ + --tensor_model_parallel_size 8 --expert_model_parallel_size 8 \ + --aiob_enable --phase decode +``` + +### A.5 Vidur 降级路径验证 + +使用 DeepSeek-671B tp=8 PD 分离配置运行完整 Vidur 模拟 (场景 3 类似配置): + +| 项目 | 结果 | +|------|------| +| 退出码 | 0 (成功) | +| 请求数 | 3/3 完成 | +| PD 分离 | 正常 (prefill replica 0/1, decode replica 2/3) | +| 执行时间数据 | 完整 (request_e2e_time, execution_time 等均有值) | +| 结果文件 | request_metrics.csv, config.json, chrome_trace.json | + +结论: Vidur 在 AICB prefill profiling 受限的情况下仍能正常完成模拟，降级路径可靠。 + +### A.6 适用范围 + +此问题**仅影响**以下特定组合: + +- **GPU**: H20 (SM90 架构) +- **模型**: DeepSeek-V3-671B (num_attention_heads=128) +- **阶段**: prefill (使用 `flash_mla_sparse_fwd` kernel) +- **并行度**: tp >= 4 (h_q = 128/tp < 64) + +**不受影响**: +- 其他 GPU 架构 (非 SM90) +- 其他模型 (Qwen3-MoE-235B, Qwen3-Next-80B 等) +- Decode 阶段 (使用不同的 attention kernel) +- tp <= 2 的配置 (h_q >= 64, 满足对齐要求) + +### A.7 测试环境 + +| 项目 | 版本 | +|------|------| +| GPU | NVIDIA H20-3e × 8 (143771 MiB each) | +| GPU Driver | 570.133.20 | +| CUDA | 12.9 | +| Python | 3.10.19 (vidur conda env) | +| PyTorch | 2.8.0 | +| FlashMLA | 1.0.0+1408756 | +| FlashInfer | 0.2.5 | +| AICB | commit `23eec3c48ca2d2d93dd888a4c7b22ab4421e782f` | +| conda env | vidur | diff --git a/vidur-alibabacloud/vidur/config/config.py b/vidur-alibabacloud/vidur/config/config.py index 3fe44e99..634faff1 100644 --- a/vidur-alibabacloud/vidur/config/config.py +++ b/vidur-alibabacloud/vidur/config/config.py @@ -509,18 +509,16 @@ class ReplicaConfig: ) # ============================================================ - # [EP Auto] Auto-computed EP, user doesn't need to specify; user value will be overridden - # Temporary = tp * pp (per-replica), later overridden in cluster.py - # to tp * pp * dp (full cluster world_size) - # [EP Auto] EP 自动计算, 用户无需手动传, 传了也会被覆盖 - # 临时值 = tp * pp (per-replica), 后续在 cluster.py 中 - # 覆盖为 tp * pp * dp (全集群 world_size) + # [EP] EP 不支持手动指定。当前实现中 EP 会被自动覆盖为 cluster world_size + # (tp×pp×dp)，在 cluster.py 中执行覆盖。若用户传入的值 != world_size， + # 将抛出 ValueError。 + # [EP] EP is not user-configurable. It is auto-set to cluster world_size + # (tp*pp*dp) in cluster.py. ValueError raised if user value != world_size. # ============================================================ expert_model_parallel_size: int = field( default=1, - metadata={"help": "Degree of expert model parallelism. " - "Auto-computed as cluster world_size (tp*pp*dp) in cluster.py, " - "user-provided value will be overridden."}, + metadata={"help": "Auto-set to world_size in cluster.py. " + "User value != world_size raises ValueError."}, ) # ============================================================ @@ -565,19 +563,16 @@ def __post_init__(self): self.world_size = self.num_pipeline_stages * self.tensor_parallel_size # ============================================================ - # [EP] Temporary = tp * pp (per-replica), excluding dp - # Will be overridden in cluster.py to tp * pp * dp (full cluster world_size) - # Initialization only, printed value is for reference - # [EP] 临时值 = tp * pp (per-replica), 不含 dp - # 后续在 cluster.py 中会被覆盖为 tp * pp * dp (全集群 world_size) - # 这里只是初始化, 打印仅供参考 + # [EP] Reject user-specified EP != world_size, then auto-set + # [EP] 拒绝用户传入的 EP != world_size，然后自动设置 # ============================================================ - user_ep = self.expert_model_parallel_size - self.expert_model_parallel_size = self.world_size # Temporary, overridden in cluster.py / 临时值, cluster.py 会覆盖 - if user_ep != 1 and user_ep != self.world_size: - logger.debug(f"[EP] Note: user-provided expert_model_parallel_size={user_ep}, " - f"temporarily set to per-replica ws={self.world_size}, " - f"final value will be overridden in cluster.py as tp*pp*dp") + if self.expert_model_parallel_size != 1 and self.expert_model_parallel_size != self.world_size: + raise ValueError( + f"EP 不支持手动指定。当前 EP={self.expert_model_parallel_size}，" + f"但 world_size={self.world_size}。EP 会在运行时自动设为 world_size。" + f"请移除 --replica_config_expert_model_parallel_size 参数或设为 {self.world_size}。" + ) + self.expert_model_parallel_size = self.world_size # Auto-set, overridden again in cluster.py / 自动设置, cluster.py 会再次覆盖 # 打印 ReplicaConfig 配置摘要 | Print ReplicaConfig summary logger.debug(f"[ReplicaConfig] tp={self.tensor_parallel_size}, pp={self.num_pipeline_stages}, " @@ -736,10 +731,15 @@ class BaseExecutionTimePredictorConfig(BasePolyConfig): metadata={"help": "Path to the simai config file."}, ) aicb_force_bs1: bool = field( - default=True, - metadata={"help": "AICB safe mode: always use bs=1 for CSV generation. " - "Avoids per_token_group_quant_fp8 CUDA kernel incompatibility " - "with bs>1 on some GPUs (e.g. H20). Set to False to use actual batch size."}, + default=False, + metadata={"help": "AICB fallback switch: force bs=1 for CSV generation. " + "Default OFF. " + "Known limitation: DeepSeek-V3-671B prefill AICB profiling fails on " + "H20 (SM90) when tp>=4, due to FlashMLA flash_mla_sparse_fwd kernel's " + "h_q alignment requirement (B_H=64). This does not affect decode or " + "other models. The simulator will automatically fall back when CSV is " + "unavailable. Set to True only if you encounter CUDA errors during " + "AICB profiling."}, ) diff --git a/vidur-alibabacloud/vidur/config/device_sku_config.py b/vidur-alibabacloud/vidur/config/device_sku_config.py index f6204f87..01971976 100644 --- a/vidur-alibabacloud/vidur/config/device_sku_config.py +++ b/vidur-alibabacloud/vidur/config/device_sku_config.py @@ -30,7 +30,6 @@ class A100DeviceSKUConfig(BaseDeviceSKUConfig): @staticmethod def get_type(): - # import pdb; pdb.set_trace() # > return DeviceSKUType.A100 @dataclass @@ -41,7 +40,6 @@ class H20DeviceSKUConfig(BaseDeviceSKUConfig): @staticmethod def get_type(): - # import pdb; pdb.set_trace() # > return DeviceSKUType.H20 @@ -63,27 +61,3 @@ class H800DeviceSKUConfig(BaseDeviceSKUConfig): @staticmethod def get_type(): return DeviceSKUType.H800 - -# NOTE: untested, for reference only (未经测试，仅供参考) -# @dataclass -# class H200DeviceSKUConfig(BaseDeviceSKUConfig): -# fp16_tflops: int = 989 -# fp8_tflops: int = 1979 -# total_memory_gb: int = 141 -# -# @staticmethod -# def get_type(): -# return DeviceSKUType.H200 - - -# GB200 NVL72 -# NOTE: untested, for reference only (未经测试，仅供参考) -# @dataclass -# class GB200DeviceSKUConfig(BaseDeviceSKUConfig): -# fp16_tflops: int = 2500 -# fp8_tflops: int = 5000 -# total_memory_gb: int = 192 -# -# @staticmethod -# def get_type(): -# return DeviceSKUType.GB200 \ No newline at end of file diff --git a/vidur-alibabacloud/vidur/config/node_sku_config.py b/vidur-alibabacloud/vidur/config/node_sku_config.py index 6f89c86b..425746a0 100644 --- a/vidur-alibabacloud/vidur/config/node_sku_config.py +++ b/vidur-alibabacloud/vidur/config/node_sku_config.py @@ -7,6 +7,15 @@ logger = init_logger(__name__) +# num_devices_per_node is used in sklearn_execution_time_predictor.py: +# - L72: devices_per_node = self._replica_config.node_config.num_devices_per_node +# - L73-75: Validates num_workers vs devices_per_node (legality assertion) +# - L77: Determines multi-node: self._is_multi_node = num_workers > devices_per_node +# It controls whether TP/EP communication uses NVLink (intra-node) or RDMA (inter-node). +# num_devices_per_node 在 sklearn_execution_time_predictor.py 中被引用: +# - L72: 获取 devices_per_node +# - L73-75: 校验 num_workers 与 devices_per_node 的合法性 +# - L77: 判断是否跨节点通信 (NVLink vs RDMA) @dataclass class BaseNodeSKUConfig(BaseFixedConfig): num_devices_per_node: int diff --git a/vidur-alibabacloud/vidur/entities/execution_time.py b/vidur-alibabacloud/vidur/entities/execution_time.py index 746e4a11..4104c6e8 100644 --- a/vidur-alibabacloud/vidur/entities/execution_time.py +++ b/vidur-alibabacloud/vidur/entities/execution_time.py @@ -770,11 +770,13 @@ def _generate_or_find_bs1_csv( [AICB Safe Mode] Always use bs=1 to generate or find CSV. [AICB Safe Mode] 始终使用 bs=1 生成或查找 CSV - Reason: AICB's per_token_group_quant_fp8 CUDA kernel is incompatible - with bs>1 on some GPUs (e.g. H20), causing "CUDA error: invalid - configuration argument". - 原因: AICB 的 per_token_group_quant_fp8 等 CUDA kernel 在某些 GPU - (如 H20) 上对 bs>1 不兼容 + Reason: DeepSeek-V3-671B prefill AICB profiling fails on H20 (SM90) + when tp>=4, due to FlashMLA `flash_mla_sparse_fwd` kernel's h_q + alignment requirement (B_H=64). This does not affect decode or + other models. + 原因: DeepSeek-V3-671B prefill 在 H20 (SM90) 上当 tp>=4 时 AICB + profiling 失败，因 FlashMLA flash_mla_sparse_fwd kernel 的 h_q + 对齐要求 (B_H=64)。不影响 decode 或其他模型。 Strategy / 策略: 1. If requested bs=1, generate directly / 如果请求的就是 bs=1, 直接生成 @@ -885,7 +887,7 @@ def _load_aicb_data(self) -> Dict[int, Dict[str, Dict[str, float]]]: # === Step 3: Cache miss, need to read/generate CSV === # === 步骤3: 缓存未命中，需要读取/生成CSV === - print(f"[AICB] Cache miss, loading CSV (缓存未命中，需要加载CSV): " + logger.debug(f"[AICB] Cache miss, loading CSV (缓存未命中，需要加载CSV): " f"model={model_name}, bs={bs}, seq={seq}, phase={phase}") csv_path = self._get_aicb_csv_path() @@ -893,10 +895,12 @@ def _load_aicb_data(self) -> Dict[int, Dict[str, Dict[str, float]]]: if not os.path.exists(full_csv_path): if self._config.aicb_force_bs1: - # [AICB Safe Mode] Always use bs=1 to generate/find CSV - # Reason: AICB's per_token_group_quant_fp8 CUDA kernel - # is incompatible with bs>1 on some GPUs (e.g. H20) - # Strategy: Use bs=1 CSV (per-token time is independent of batch size) + # [AICB Fallback] User explicitly enabled force-bs1 mode + # This forces bs=1 CSV generation regardless of actual batch size. + # Known issue: DeepSeek-V3-671B prefill AICB profiling fails on H20 + # (SM90) when tp>=4, due to FlashMLA flash_mla_sparse_fwd kernel's + # h_q alignment requirement (B_H=64). Does not affect decode or + # other models. full_csv_path = self._generate_or_find_bs1_csv( model_name, ws, tp, pp, ep, bs, seq, phase, full_csv_path ) diff --git a/vidur-alibabacloud/vidur/events/global_schedule_event.py b/vidur-alibabacloud/vidur/events/global_schedule_event.py index 24329b49..3cef2b1e 100644 --- a/vidur-alibabacloud/vidur/events/global_schedule_event.py +++ b/vidur-alibabacloud/vidur/events/global_schedule_event.py @@ -21,12 +21,9 @@ def handle_event( ) -> List[BaseEvent]: from vidur.events.replica_schedule_event import ReplicaScheduleEvent - # import pdb; pdb.set_trace() # > - self._replica_set = set() # _request_mapping: [(replica_id, request), ...] - # import pdb; pdb.set_trace() # > self._request_mapping = scheduler.schedule() diff --git a/vidur-alibabacloud/vidur/metrics/metrics_store.py b/vidur-alibabacloud/vidur/metrics/metrics_store.py index e5247395..96e7cf67 100644 --- a/vidur-alibabacloud/vidur/metrics/metrics_store.py +++ b/vidur-alibabacloud/vidur/metrics/metrics_store.py @@ -609,10 +609,6 @@ def _on_request_end(self, time: float, request: Request) -> None: self._request_metrics_time_distributions[ RequestMetricsTimeDistributions.COMPLETED_AT ].put(request.id,request._completed_at) - # print(f"> Debug: after entry (self._request_metrics_time_distributions[RequestMetricsTimeDistributions.COMPLETED_AT]={self._request_metrics_time_distributions[RequestMetricsTimeDistributions.COMPLETED_AT]}") - # print(f"> Debug: after entry (request.id,request._completed_at)={(request.id,request._completed_at)}") - # import pdb; pdb.set_trace() # > - self._request_metrics_time_distributions[ RequestMetricsTimeDistributions.PREEMPTED_TIME ].put(request.id,request._preempted_time) diff --git a/vidur-alibabacloud/vidur/scheduler/global_scheduler/lor_global_scheduler.py b/vidur-alibabacloud/vidur/scheduler/global_scheduler/lor_global_scheduler.py index 9373cd01..92ce3eef 100644 --- a/vidur-alibabacloud/vidur/scheduler/global_scheduler/lor_global_scheduler.py +++ b/vidur-alibabacloud/vidur/scheduler/global_scheduler/lor_global_scheduler.py @@ -62,7 +62,4 @@ def schedule(self) -> List[Tuple[int, Request]]: replica_id = min(pending_requests_map.items(), key=lambda x: x[1])[0] pending_requests_map[replica_id] += 1 request_mapping.append((replica_id, request)) - - # import pdb; pdb.set_trace() # > - # print(f"> Debug: {request_mapping}{pending_requests_map}") return request_mapping diff --git a/vidur-alibabacloud/vidur/scheduler/replica_scheduler/sarathi_replica_scheduler.py b/vidur-alibabacloud/vidur/scheduler/replica_scheduler/sarathi_replica_scheduler.py index 0fd426a8..c033a188 100644 --- a/vidur-alibabacloud/vidur/scheduler/replica_scheduler/sarathi_replica_scheduler.py +++ b/vidur-alibabacloud/vidur/scheduler/replica_scheduler/sarathi_replica_scheduler.py @@ -84,7 +84,6 @@ def _get_request_next_num_tokens( return next_num_tokens def _get_next_batch(self) -> Batch: - # import pdb; pdb.set_trace() # > requests = [] num_tokens = [] skipped_requests = [] diff --git a/vidur-alibabacloud/vidur/utils/base_registry.py b/vidur-alibabacloud/vidur/utils/base_registry.py index 5dc0731e..f54bb2a4 100644 --- a/vidur-alibabacloud/vidur/utils/base_registry.py +++ b/vidur-alibabacloud/vidur/utils/base_registry.py @@ -57,5 +57,4 @@ def get_key_from_str(cls, key_str: str) -> BaseIntEnum: @classmethod def get_from_str(cls, key_str: str, *args, **kwargs) -> Any: - # import pdb; pdb.set_trace() # > return cls.get(cls.get_key_from_str(key_str), *args, **kwargs) From ae470248d5db1ac60a7817a1636c2bb64fe4222c Mon Sep 17 00:00:00 2001 From: tianhao909 <843101550@qq.com> Date: Tue, 21 Apr 2026 05:54:27 +0000 Subject: [PATCH 5/8] fix(vidur-alibabacloud): complete bs=8 verification matrix + FlashMLA constraint analysis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [EN] - Run missing bs=8 test cases for DeepSeek-671B (tp=4/8) and Qwen3-Next-80B (tp=1) - Replace all dash marks with actual test results and explanations - Add FlashMLA h_q alignment constraint analysis with version pinning evidence (A.8) - Archive experiment logs inline as Appendix A.9 for reproducibility - Update environment baseline (CUDA 12.8, PyTorch 2.8.0+cu128, vLLM 0.11.0) [ZH] - 补测 DeepSeek-671B (tp=4/8) 和 Qwen3-Next-80B (tp=1) 的 bs=8 用例 - 将所有未解释标记替换为实际测试结果并附加说明 - 新增 FlashMLA h_q 对齐约束分析（含版本锁定证据）(A.8) - 内联实验日志作为附录 A.9 以确保可复现性 - 更新环境基线信息 (CUDA 12.8, PyTorch 2.8.0+cu128, vLLM 0.11.0) --- .../tests/test_tutorial_report_cn.md | 124 +++++++++++++++++- 1 file changed, 117 insertions(+), 7 deletions(-) diff --git a/vidur-alibabacloud/tests/test_tutorial_report_cn.md b/vidur-alibabacloud/tests/test_tutorial_report_cn.md index 95fc7d1e..5067ada9 100644 --- a/vidur-alibabacloud/tests/test_tutorial_report_cn.md +++ b/vidur-alibabacloud/tests/test_tutorial_report_cn.md @@ -295,11 +295,15 @@ Vidur (vidur.main) |----|-----------|------|------|------|------|------| | 1 | 128 | ✅ | ✅ | ✅ | ✅ | h_q=128, 128%64=0, 全部通过 | | 2 | 64 | ✅ | ✅ | ✅ | ✅ | h_q=64, 64%64=0, 边界值通过 | -| 4 | 32 | ❌ | ❌ | ❌ | — | h_q=32, 32%64≠0, 全部失败 | -| 8 | 16 | ❌ | ❌ | ❌ | — | h_q=16, 16%64≠0, 全部失败 | +| 4 | 32 | ❌ | ❌ | ❌ | ❌ | h_q=32, 32%64≠0, 全部失败 | +| 8 | 16 | ❌ | ❌ | ❌ | ❌ | h_q=16, 16%64≠0, 全部失败 | -**关键结论**: 失败完全由 tp 决定（h_q 对齐），与 bs 无关。tp=1/2 下 bs=2/4/8 全部通过， -tp=4/8 下即使 bs=1 也会失败。之前 "bs>1 fails" 的描述不准确。 +> **bs=8 补测说明** (2026-04-21): tp=4/8 的 bs=8 已补测验证，错误签名与 bs=1/2/4 完全一致 +> (`params.h_q % B_H == 0`)，确认失败原因相同。原 "—" 标记已替换为实际测试结果 ❌。 +> 详细实验日志见附录 A.9。 + +**关键结论**: 失败完全由 tp 决定（h_q 对齐），与 bs 无关。tp=1/2 下 bs=1/2/4/8 全部通过， +tp=4/8 下 bs=1/2/4/8 全部失败。矩阵中所有格子均已实测验证，无未解释标记。 **Decode 测试 (不受影响验证)** @@ -367,10 +371,116 @@ cd aicb && conda run -n vidur python -m workload_generator.Vidur_workload_genera |------|------| | GPU | NVIDIA H20-3e × 8 (143771 MiB each) | | GPU Driver | 570.133.20 | -| CUDA | 12.9 | +| CUDA | 12.8 | | Python | 3.10.19 (vidur conda env) | -| PyTorch | 2.8.0 | -| FlashMLA | 1.0.0+1408756 | +| PyTorch | 2.8.0+cu128 | +| FlashMLA | 1.0.0+1408756 (pinned commit `1408756a88e52a25196b759eaf8db89d2b51b5a1`) | | FlashInfer | 0.2.5 | | AICB | commit `23eec3c48ca2d2d93dd888a4c7b22ab4421e782f` | +| vLLM | 0.11.0 | | conda env | vidur | + +### A.8 FlashMLA `h_q % B_H` 约束分析 + +> 补充日期: 2026-04-21 + +#### 第一层: AICB pinned 版本中的事实 + +| 组件 | 版本 | 关键代码位置 | +|------|------|-------------| +| FlashMLA | 1.0.0+1408756 | `csrc/sm90/prefill/sparse/config.h` → `B_H = 64` | +| AICB | 23eec3c | `AiobDeepSeek.py:182` → `h_q = self.num_heads // self.tp` | +| vLLM | 0.11.0 | `requirements.txt` pinned 依赖 | + +**约束逻辑**: FlashMLA 的 SM90 prefill sparse kernel 以 `B_H=64` 为 WGMMA tile 大小， +运行时要求 `params.h_q % B_H == 0`。AICB 按 `h_q = num_attention_heads / tp` 计算每个 +TP rank 的 head 数量。对于 DeepSeek-V3 (`num_attention_heads=128`): + +| tp | h_q | h_q % 64 | 结果 | +|----|-----|----------|------| +| 1 | 128 | 0 | PASS | +| 2 | 64 | 0 | PASS | +| 4 | 32 | 32 | FAIL | +| 8 | 16 | 16 | FAIL | + +#### 第二层: 上游最新版本观察 + +- FlashMLA main 分支 (截至 2026-04-21): `B_H=64` 未变 (`config.h`) +- 代码结构重构: 断言从 `fwd.cu:647` 移至 `phase1.cuh`，但约束逻辑不变 +- 上游 PR #150 (2026-01-16) 做了多处重构，但 B_H 值未修改 + +**结论**: 上游最新版本仍有此约束，非 pinned 版本特有问题。 + +#### 第三层: 仿真与真实一致性 + +- AICB 的 `h_q = num_heads // tp` 与真实 vLLM 推理时的 head 分配逻辑一致 +- 真实 vLLM 在 tp>=4 时也会触发同样的 FlashMLA 断言 +- **结论**: 仿真行为 = 真实行为（都会触发），仿真是准确的 + +### A.9 bs=8 补测实验日志 (2026-04-21) + +> 补测目的: 消除 A.4 验证矩阵中 tp=4/8 bs=8 的 "—" 标记，并验证 Qwen3-Next-80B bs=8 + +#### Case 1: Qwen3-Next-80B prefill tp=1 bs=8 — PASS + +```bash +# 命令 +cd aicb && conda run -n vidur python -m workload_generator.Vidur_workload_generator \ + Qwen3-Next-80B ./scripts/inference_configs/qwen3_next_default.json \ + --seq_length 1024 --micro_batch 8 --world_size 1 \ + --tensor_model_parallel_size 1 --expert_model_parallel_size 1 \ + --aiob_enable --phase prefill +``` + +- **退出码**: 0 (成功) +- **输出**: `results/workload/vidur-Qwen3-Next-80B-world_size1-tp1-pp1-ep1-bs8-seq1024-prefill` +- **说明**: Qwen3-Next-80B 使用 FlashInfer (非 FlashMLA)，不受 B_H=64 约束 + +#### Case 2: DeepSeek-671B prefill tp=4 bs=8 — FAIL + +```bash +# 命令 +cd aicb && conda run -n vidur python -m workload_generator.Vidur_workload_generator \ + DeepSeek-671B ./scripts/inference_configs/deepseek_default.json \ + --seq_length 1024 --micro_batch 8 --world_size 4 \ + --tensor_model_parallel_size 4 --expert_model_parallel_size 4 \ + --aiob_enable --phase prefill +``` + +- **退出码**: 1 (失败) +- **错误签名**: + ``` + Assertion failed (/tmp/pip-req-build-gg88vm1n/csrc/sm90/prefill/sparse/fwd.cu:647): params.h_q % B_H == 0 + ``` +- **根因**: h_q = 128/4 = 32, 32 % 64 ≠ 0 +- **与 bs=1/2/4 一致**: 错误签名完全相同，确认失败由 tp 决定 + +#### Case 3: DeepSeek-671B prefill tp=8 bs=8 — FAIL + +```bash +# 命令 +cd aicb && conda run -n vidur python -m workload_generator.Vidur_workload_generator \ + DeepSeek-671B ./scripts/inference_configs/deepseek_default.json \ + --seq_length 1024 --micro_batch 8 --world_size 8 \ + --tensor_model_parallel_size 8 --expert_model_parallel_size 8 \ + --aiob_enable --phase prefill +``` + +- **退出码**: 1 (失败) +- **错误签名**: + ``` + Assertion failed (/tmp/pip-req-build-gg88vm1n/csrc/sm90/prefill/sparse/fwd.cu:647): params.h_q % B_H == 0 + ``` +- **根因**: h_q = 128/8 = 16, 16 % 64 ≠ 0 +- **与 bs=1/2/4 一致**: 错误签名完全相同，确认失败由 tp 决定 + +#### 补测结论 + +| Case | 模型 | tp | bs | 预期 | 实际 | 匹配 | +|------|------|----|----|------|------|------| +| 1 | Qwen3-Next-80B | 1 | 8 | PASS | PASS | ✅ | +| 2 | DeepSeek-671B | 4 | 8 | FAIL | FAIL | ✅ | +| 3 | DeepSeek-671B | 8 | 8 | FAIL | FAIL | ✅ | + +全部 3 个补测 case 与预期一致。A.4 验证矩阵已更新为完整实测结果。 + From 9c948b953a3a8898906bb9ab4ae642e312e04eea Mon Sep 17 00:00:00 2001 From: tianhao909 <843101550@qq.com> Date: Tue, 21 Apr 2026 08:37:25 +0000 Subject: [PATCH 6/8] docs(test-report): add vLLM v0.11.0 source comparison to A.8 FlashMLA analysis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [EN] - Fix auto-linked #150 reference: replace bare #150 with full FlashMLA PR URL - Add vLLM v0.11.0 vs AICB call path comparison (flash_mla_sparse_prefill vs flash_mla_sparse_fwd) - Add supplementary note on latest vLLM (582340f27) head padding / FP8 mixed batch workarounds - Qualify "same kernel" statement: same underlying SM90 CUDA kernel, different Python entry points - Use version-pinned permalinks for all external references [ZH] - 修复 #150 auto-link: 将裸 #150 替换为 FlashMLA PR 完整 URL - 新增 vLLM v0.11.0 与 AICB 调用路径对照 (flash_mla_sparse_prefill vs flash_mla_sparse_fwd) - 补充最新 vLLM (582340f27) 的 head padding / FP8 mixed batch workaround 备注 - 限定 "同一 kernel" 表述: 底层 SM90 CUDA kernel 相同，Python 入口不同 - 所有外部引用改用版本固定的 permalink --- .../tests/test_tutorial_report_cn.md | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/vidur-alibabacloud/tests/test_tutorial_report_cn.md b/vidur-alibabacloud/tests/test_tutorial_report_cn.md index 5067ada9..3cebfc04 100644 --- a/vidur-alibabacloud/tests/test_tutorial_report_cn.md +++ b/vidur-alibabacloud/tests/test_tutorial_report_cn.md @@ -407,14 +407,57 @@ TP rank 的 head 数量。对于 DeepSeek-V3 (`num_attention_heads=128`): - FlashMLA main 分支 (截至 2026-04-21): `B_H=64` 未变 (`config.h`) - 代码结构重构: 断言从 `fwd.cu:647` 移至 `phase1.cuh`，但约束逻辑不变 -- 上游 PR #150 (2026-01-16) 做了多处重构，但 B_H 值未修改 +- 上游 [FlashMLA PR #150](https://github.com/deepseek-ai/FlashMLA/pull/150) (2026-01-16) 做了多处重构，但 B_H 值未修改 **结论**: 上游最新版本仍有此约束，非 pinned 版本特有问题。 +#### vLLM 源码对照 (回答 reviewer: "vllm源码怎么写的") + +**(a) vLLM v0.11.0 (AICB pinned) 中的调用路径:** + +| 层级 | 文件 | 关键内容 | +|------|------|----------| +| ops 层 | [`vllm/attention/ops/flashmla.py`](https://github.com/vllm-project/vllm/blob/v0.11.0/vllm/attention/ops/flashmla.py) | `flash_mla_sparse_prefill()` → 调用 `torch.ops._flashmla_C.sparse_prefill_fwd` | +| backend 层 | [`vllm/v1/attention/backends/mla/flashmla_sparse.py`](https://github.com/vllm-project/vllm/blob/v0.11.0/vllm/v1/attention/backends/mla/flashmla_sparse.py) (544行) | 导入 `flash_mla_sparse_prefill`，prefill 阶段直接调用 | + +- **无 head padding 机制**: h_q 直接传入 CUDA kernel，h_q < 64 → 触发 `B_H=64` 断言失败 + +**(b) AICB 的调用路径:** + +| 层级 | 文件 | 关键内容 | +|------|------|----------| +| 入口 | `AiobDeepSeek.py:235` | 调用 `flash_mla_sparse_fwd()` (直接导入 flash_mla Python 包) | + +- Python 入口函数名不同: vLLM 用 `flash_mla_sparse_prefill`，AICB 用 `flash_mla_sparse_fwd` +- **底层执行相同的 FlashMLA SM90 CUDA sparse prefill kernel** +- h_q 计算方式一致: `h_q = num_attention_heads // tp` +- **结论: AICB 仿真调用路径与 vLLM v0.11.0 的真实推理路径在底层 CUDA kernel 层面一致** + +**(c) latest vLLM 补充备注:** + +> 版本标注: vLLM main HEAD [`582340f27`](https://github.com/vllm-project/vllm/blob/582340f27/vllm/v1/attention/backends/mla/flashmla_sparse.py) (约 v0.18.2, 截至 2026-04-21) + +| DeepSeek-V3 tp | h_q | v0.11.0 行为 | main 行为 | main workaround 路径 | +|----------------|-----|------------|----------|-----------------------| +| tp=1 | 128 | PASS | PASS | 不需要 | +| tp=2 | 64 | PASS | PASS | 不需要 | +| tp=4 | 32 | **FAIL** | **PASS** | BF16 prefill + head padding (32→64) | +| tp=8 | 16 | **FAIL** | **PASS** | mixed batch FP8 decode kernel (绕过 BF16 prefill) | + +- 新增 `MIN_HEADS_FOR_BF16_PREFILL = 32` (L63) +- tp=4 (h_q=32): `32 < 32` = False → BF16 prefill + head padding 到 64 +- tp=8 (h_q=16): `16 < 32` = True → mixed batch FP8 → 绕过 BF16 prefill 约束 +- **备注: 此 workaround 不影响当前 AICB 结论**，因为 AICB pinned 的是 v0.11.0 + #### 第三层: 仿真与真实一致性 - AICB 的 `h_q = num_heads // tp` 与真实 vLLM 推理时的 head 分配逻辑一致 - 真实 vLLM 在 tp>=4 时也会触发同样的 FlashMLA 断言 +- **vLLM v0.11.0**: AICB 仿真行为 = vLLM 真实推理行为（都会在 tp>=4 触发断言） +- **vLLM main (582340f27)**: 已通过两种方式规避了此问题： + - tp=4 (h_q=32): 在调用 kernel 前将 h_q 从 32 填充到 64 (head padding)，使其满足 `h_q % 64 == 0`，不再触发断言 + - tp=8 (h_q=16): 切换到 FP8 mixed batch decode kernel，完全绕过了 BF16 sparse prefill 路径，不会触发 B_H=64 约束 + - **但这不影响当前 AICB 结论**，因为 AICB pinned 的是 v0.11.0，该版本没有以上 workaround - **结论**: 仿真行为 = 真实行为（都会触发），仿真是准确的 ### A.9 bs=8 补测实验日志 (2026-04-21) From 3479f8b3cf1184a512084133e95e98d31ce67c5a Mon Sep 17 00:00:00 2001 From: tianhao909 <843101550@qq.com> Date: Wed, 22 Apr 2026 10:09:57 +0000 Subject: [PATCH 7/8] docs(readme): simplify 1.6 release notes + trilingual sync + terminology unification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [EN] - Condense SimAI 1.6 release notes to 3 key points: GPU memory, decode interpolation, PD disaggregation - Update release date to Apr 23, 2026 (confirmed release date) - Unify terminology: PD Separation → PD Disaggregation across all READMEs - Full sync README.ja.md with all missing sections - Add trilingual language switcher links [ZH] - SimAI 1.6 release notes 精简为 3 点：GPU 显存建模、decode 插值、PD 分离 - 发布日期更新为 Apr 23, 2026 - 术语统一：PD Separation → PD Disaggregation - README.ja.md 全量同步补齐所有缺失章节 - 三语语言切换链接互通 --- README.ja.md | 149 ++++++++++++++++++++++++++++------- README.md | 15 ++-- README_CN.md | 15 ++-- vidur-alibabacloud/README.md | 12 +-- 4 files changed, 139 insertions(+), 52 deletions(-) diff --git a/README.ja.md b/README.ja.md index dd96f29f..5539f64c 100644 --- a/README.ja.md +++ b/README.ja.md @@ -1,6 +1,36 @@ +

+ 中文 ｜ English ｜ 日本語 +

+ +# SimAI + +[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) +[![NSDI'25](https://img.shields.io/badge/NSDI'25-SimAI-blue.svg)](https://ennanzhai.github.io/pub/nsdi25spring-simai.pdf) + # 最新ニュース -### SimCCLのアップデート -[2025/06] SimCCLのコードが最初に[SimCCL](https://github.com/aliyun/SimAI/tree/SimCCL)ブランチで公開され、まもなくSimCCLリポジトリでリリースされます。 + +### 最近のアップデート + +- [2026/04] **SimAI 1.6 リリース！** 主な更新： + - 推論シミュレーション向け GPU メモリモデリング（パラメータカウント＆KV Cache）。 + - Decode 時間推定の線形補間（最近傍探索の代替）。 + - PD Disaggregation メモリプランニング（Prefill/Decode 独立バジェット）。 + +- [2025/12] **SimAI 1.5 リリース！** このリリースでは、マルチリクエスト**推論**ワークロード向けのエンドツーエンドシミュレーションが実現されました。主な機能： + + - **高度な推論シミュレーション：** Prefill/Decode 分離を用いた複雑なシナリオのモデリング。 + - **最新モデルサポート：** DeepSeek、Qwen3Moe、Qwen3Next に対応。詳細は [AICB の README](./aicb/README.md) を参照してください。 + - **リクエストスケジューリング：** リクエストスケジューリングは、Microsoft の [Vidur](https://github.com/microsoft/vidur) から適応したコンポーネントによって処理されます。詳細は [Vidur-Alibabacloud の README](./vidur-alibabacloud/README.md) を参照してください。 + +- [2025/11] [AICB](https://github.com/aliyun/aicb/tree/master) が **DeepSeek**、**Qwen3-MoE**、**Qwen3-Next** 向けの **prefill/decode** 推論ワークロード生成に対応しました。 + +- [2025/09] [AICB](https://github.com/aliyun/aicb/tree/master) が DeepSeek 向けのトレーニングワークロード生成に対応しました。[@parthpower](https://github.com/parthpower) 氏のコントリビューションに感謝します。 + +- [2025/06] SimCCLのコードが最初に[SimCCL](https://github.com/aliyun/SimAI/tree/SimCCL)ブランチで公開され、まもなくSimCCLリポジトリでリリースされます。 + +**コミュニティからの貢献を歓迎します！** SimAI の未来を一緒に作りたい方は、お気軽に Issue を開いてアイデアを議論したり、プルリクエストを送信してください。 + +
🎯 イベントとコミュニティ活動 🎯 @@ -8,43 +38,54 @@ | 日付 | イベント | 場所 | 内容 | 形式 | |:----:|:------|:---------|:--------|:----:| -| 未定 | SimAI 2.0 | 🌐 オンライン | SimAI 2.0のリリース | 💻 バーチャル | +| -- | | | | | ### 🌟 過去のイベント -| 日付 | イベント | 場所 | 内容 | 形式 | -|:----:|:------|:---------|:--------|:----:| -| 2025年6月4日 | SimAIコミュニティ第1回ワークショップ | 📍 北京大学 | コミュニティ貢献者による3つの講演 | 🎓 現地 | -| 2025年5月24日 | 第28回Chinasysワークショップ | 📍 重慶大学 | SimAIに関する招待講演 | 🎓 現地 | -| 2024年12月27日 | SimAI技術発表会 | 📍 北京航空航天大学 | SimAI技術共有とディスカッション | 🎓 現地 | -| 2024年12月6日 | HKUST技術ワークショップ | 📍 香港科技大学(広州) | SimAI技術共有とディスカッション | 🎓 現地 | -| 2024年12月5日 | [Bench'24カンファレンス](https://mp.weixin.qq.com/s/STic_E12xMhZRxhzK9wRnw) | 📍 広州 | SimAIチュートリアルと詳細セッション | 🎓 現地 | -| 2024年11月26日 | SimAIコミュニティライブストリーム | 🌐 オンライン | インタラクティブな技術ディスカッションとデモ（400人以上参加） | 💻 バーチャル | -| 2024年11月15日 | 技術ワークショップ | 📍 千島湖 | SimAIオフライン技術交流会 | 🎯 現地 | -| 2024年10月18日 | ゲスト講義 | 📍 復旦大学 | SimAIチュートリアルと公開講座 | 🎓 現地 | -| 2024年9月24-26日 | CCF HPC China 2024 | 📍 武漢 | SimAI紹介と技術発表 | 🎤 カンファレンス | +| 日付 | イベント | 場所 | 内容 | 形式 | +|:----------------:|:------------------------------------------------------------------------ |:----------------------- |:-------------------------------------------------------- |:-------------:| +| 2026年4月23日 | SimAI 1.6 | 🌐 オンライン | SimAI 1.6 のリリース | 💻 バーチャル | +| 2025年12月30日 | SimAI 1.5 | 🌐 オンライン | SimAI 1.5 のリリース | 💻 バーチャル | +| 2025年6月4日 | SimAIコミュニティ第1回ワークショップ | 📍 北京大学 | コミュニティ貢献者による3つの講演 | 🎓 現地 | +| 2025年5月24日 | 第28回Chinasysワークショップ | 📍 重慶大学 | SimAIに関する招待講演 | 🎓 現地 | +| 2024年12月27日 | SimAI技術発表会 | 📍 北京航空航天大学 | SimAI技術共有とディスカッション | 🎓 現地 | +| 2024年12月6日 | HKUST技術ワークショップ | 📍 香港科技大学(広州) | SimAI技術共有とディスカッション | 🎓 現地 | +| 2024年12月5日 | [Bench'24カンファレンス](https://mp.weixin.qq.com/s/STic_E12xMhZRxhzK9wRnw) | 📍 広州 | SimAIチュートリアルと詳細セッション | 🎓 現地 | +| 2024年11月26日 | SimAIコミュニティライブストリーム | 🌐 オンライン | インタラクティブな技術ディスカッションとデモ（400人以上参加）| 💻 バーチャル | +| 2024年11月15日 | 技術ワークショップ | 📍 千島湖 | SimAIオフライン技術交流会 | 🎯 現地 | +| 2024年10月18日 | ゲスト講義 | 📍 復旦大学 | SimAIチュートリアルと公開講座 | 🎓 現地 | +| 2024年9月24-26日 | CCF HPC China 2024 | 📍 武漢 | SimAI紹介と技術発表 | 🎤 カンファレンス | +
--- +## ドキュメント + +詳細なドキュメントは [チュートリアル](./docs/Tutorial.md) を参照してください。 + +--- + # 目次 + - [SimAI 概要](#simai-概要) - [はじめに](#はじめに) - [コンポーネント](#コンポーネント) - [シナリオ](#シナリオ) - [引用](#引用) -- [使い方](#使い方) +- [クイックスタート](#クイックスタート) - [セットアップ](#セットアップ) - - [ソースコードから](#ソースコードから) - [SimAI-Analyticalの使い方](#simai-analyticalの使い方) - [SimAI-Simulationの使い方](#simai-simulationの使い方) + - [マルチリクエスト推論シミュレーションの使い方](#マルチリクエスト推論シミュレーションの使い方) # SimAI 概要 + ## はじめに -**SimAI**は、業界初のフルスタック・高精度な大規模AIトレーニング用**Sim**ulator（**シミュレーター**）です。フレームワーク、集合通信、ネットワーク層など、LLMトレーニングプロセス全体を詳細にモデリング・シミュレーションします。この包括的なアプローチにより、エンドツーエンドのパフォーマンスデータが提供され、研究者は以下のことが可能になります： +**SimAI**は、業界初のフルスタック・高精度な大規模AI**推論**および**トレーニング**用**Sim**ulator（**シミュレーター**）です。フレームワーク、集合通信、ネットワーク層など、LLMトレーニングプロセス全体を詳細にモデリング・シミュレーションします。この包括的なアプローチにより、エンドツーエンドのパフォーマンスデータが提供され、研究者は以下のことが可能になります： -- トレーニングプロセスの詳細分析 +- 推論/トレーニングプロセスの詳細分析 - 特定条件下でのAIタスクの時間消費の評価 - 以下を含む様々なアルゴリズム最適化によるE2Eパフォーマンスゲインの評価： - フレームワークのパラメータ設定 @@ -63,9 +104,10 @@ SimAI --|--- SimCCL |--- astra-sim-alibabacloud |--- ns-3-alibabacloud + |--- vidur-alibabacloud -純粋なシミュレーション能力を基盤に、SimAIは4つのコンポーネント（[aicb](https://github.com/aliyun/aicb)、[SimCCL](https://github.com/aliyun/SimCCL)、[astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)、[ns-3-alibabacloud](https://github.com/aliyun/ns-3-alibabacloud)）からなる多機能なフルスタックツールキットに進化しました。これらのコンポーネントは、様々な方法で組み合わせて異なる機能を実現できます。以下に、SimAIの6つの主な使用シナリオを示します。この強力なツールでさらに多くの可能性を探求することをお勧めします。 +純粋なシミュレーション能力を基盤に、SimAIは4つのコンポーネント（[aicb](https://github.com/aliyun/aicb)、[SimCCL](https://github.com/aliyun/SimCCL)、[astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)、[ns-3-alibabacloud](https://github.com/aliyun/ns-3-alibabacloud)）からなる多機能なフルスタックツールキットに進化しました。これらのコンポーネントは、様々な方法で組み合わせて異なる機能を実現できます。以下に、SimAIの主な使用シナリオを示します。この強力なツールでさらに多くの可能性を探求することをお勧めします。 以下はSimAIシミュレータのアーキテクチャ図です： ![SimAI_Arc](./docs/images/SimAI_Arc.png) @@ -85,12 +127,12 @@ SimAIは、さまざまなシミュレーション要件を満たすために、 | シナリオ | 説明 | コンポーネントの組み合わせ | |----------|-------------|------------------------| | 1. AICBテストスイート | AICBテストスイートを使用してGPUクラスタで通信パターンを実行 | [AICB](https://github.com/aliyun/aicb) | -| 2. AICB/AIOBワークロード | トレーニングプロセスの計算/通信パターンをモデル化してワークロードを生成 | [AICB](https://github.com/aliyun/aicb) | +| 2. AICB/AIOBワークロード | **推論**/トレーニングプロセスの計算/通信パターンをモデル化してワークロードを生成 | [AICB](https://github.com/aliyun/aicb) | | 3. 集合通信分析 | 集合通信操作をポイントツーポイント通信セットに分解 | [SimCCL](https://github.com/aliyun/SimCCL) | | 4. GPUなしでの集合通信 | 非GPUクラスタでRDMA集合通信トラフィックを実行 | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(physical) | | 5. SimAI-Analytical | 任意のサーバーで迅速なAICBワークロード分析とシミュレーションを実施（基盤となるネットワークの詳細は無視） | [AICB](https://github.com/aliyun/aicb) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(analytical) | | 6. SimAI-Simulation | 任意のサーバーで完全なシミュレーションを実行 | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(simulation) + [ns-3-alibabacloud](https://github.com/aliyun/ns-3-alibabacloud) | - +| 7. マルチリクエスト推論シミュレーション | 1台のGPUサーバーを使用してマルチリクエスト**推論**のフルシミュレーションを実行 | [AICB](https://github.com/aliyun/aicb) + [SimCCL](https://github.com/aliyun/SimCCL) + [vidur-alibabacloud](./vidur-alibabacloud) + [astra-sim-alibabacloud](https://github.com/aliyun/SimAI/tree/master/astra-sim-alibabacloud)(analytical/simulation) | ## 引用 @@ -133,18 +175,18 @@ $ ./scripts/build.sh -c analytical # SimAI-Simulation (ns3)をコンパイル $ ./scripts/build.sh -c ns3 - ``` ## SimAI-Analyticalの使い方 ```bash -$ ./bin/SimAI_analytical -w example/workload_analytical.txt -g 9216 -g_p_s 8 -r test- -busbw example/busbw.yaml +$ ./bin/SimAI_analytical -w example/workload_analytical.txt -g 9216 -g_p_s 8 -r test- -busbw example/busbw.yaml ``` バス帯域幅を自動で計算するには、次のコマンドを試してください： + ```bash -$ ./bin/SimAI_analytical -w ./example/workload_analytical.txt -g 9216 -nv 360 -nic 48.5 -n_p_s 8 -g_p_s 8 -r example- +$ ./bin/SimAI_analytical -w ./example/workload_analytical.txt -g 9216 -nv 360 -nic 48.5 -n_p_s 8 -g_p_s 8 -r example- ``` ## SimAI-Simulationの使い方 @@ -155,18 +197,69 @@ $ python3 ./astra-sim-alibabacloud/inputs/topo/gen_Topo_Template.py -topo Spectr # 実行 $ AS_SEND_LAT=3 AS_NVLS_ENABLE=1 ./bin/SimAI_simulator -t 16 -w ./example/microAllReduce.txt -n ./Spectrum-X_128g_8gps_100Gbps_A100 -c astra-sim-alibabacloud/inputs/config/SimAI.conf +``` +## マルチリクエスト推論シミュレーションの使い方 + +詳細については、`vidur-alibabacloud` ディレクトリ内の [README](./vidur-alibabacloud/README.md) ファイルを参照してください。このモジュールは AICB を活用して**推論**ワークロードの計算時間をプロファイリングします。DeepGEMM や FlashMLA などの特定のハードウェアアクセラレーションライブラリに依存するため、**Hopper (SM90)** および **Blackwell (SM100)** アーキテクチャベースの NVIDIA GPU のみに対応しています。 + +```bash +# Dockerfile からビルド +docker build -t image:latest . +docker run --gpus all -it --rm image:latest +``` + +**注意：** Hopper GPU を使用する場合は、Dockerfile に `ENV FLASH_MLA_DISABLE_SM100=1` を追加してください。 + +サポートされているすべての推論シナリオ（Qwen3-Next-80B、DeepSeek-671B、Qwen3-MoE-235B）を迅速に検証するには、同梱の4シナリオテストスイートを使用してください： + +```bash +# 前提条件: conda activate vidur +bash vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh --all +# または単一シナリオを実行: +bash vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh --scenario 1 ``` +> **前提条件：** `conda activate vidur` 環境が必要です。詳細は [環境セットアップ](./vidur-alibabacloud/README.md#-environment-setup) を参照してください。 +> +> シナリオ設定テーブルと出力ファイルの詳細については、[Vidur-AlibabaCloud README](./vidur-alibabacloud/README.md#4-scenario-configuration) を参照してください。 + +# 謝辞 + +このプロジェクトに貢献してくださった以下の方々と組織に深く感謝いたします： + +- TianHao Fu（北京大学）および [TELOS-syslab](https://github.com/TELOS-syslab/) +- Parth Parikh（KEYSIGHT） +- Sarah-Michelle Hammer & Ziyi Wang（TU-Berlin） +- Xinyue Li（BUPT） +- Tong Chen（浙江大学） +- Ming Wang（BUPT） +- Tao Jiang（中国科学院計算技術研究所） + +...その他、コミュニティからの多くの個人コントリビューター（[Contributors to aliyun/SimAI](https://github.com/aliyun/SimAI/graphs/contributors) を参照）。 + +Chenning Li（MIT CSAIL）にも感謝いたします。革新的な新しいシミュレーター [M4](https://github.com/netiken/m4) への SimAI 統合の協力を開始してくれました。 + +**このプロジェクトは引き続き、より多くの貢献と提案を歓迎しています。** + +# コントリビューションガイド + +すべての貢献を歓迎します！始める前に、以下のガイドをお読みください： + +| | | +|---|---| +| [コントリビューションガイド](./CONTRIBUTING.md) | Issue やプルリクエストの提出方法 | +| [セキュリティポリシー](./SECURITY.md) | セキュリティ脆弱性の報告方法 | +| [行動規範](./CODE_OF_CONDUCT.md) | 私たちのコミュニティ基準 | +| [変更履歴](./CHANGELOG.md) | v1.5 以降のバージョン履歴 | + # お問い合わせ -ご不明な点がございましたら、Gang Lu (yunding.lg@alibaba-inc.com) または Qingxu Li (qingxu.lqx@alibaba-inc.com) までメールでお問い合わせください。 +ご不明な点がございましたら、Gang Lu (yunding.lg@alibaba-inc.com)、Feiyang Xue (xuefeiyang.xfy@alibaba-inc.com) または Qingxu Li (qingxu.lqx@alibaba-inc.com) までメールでお問い合わせください。 SimAIコミュニティのチャットグループへの参加を歓迎します。左がDingTalkグループ、右がWeChatグループです。
SimAI DingTalk - SimAI WeChat + SimAI WeChat
- -
diff --git a/README.md b/README.md index d2b2aba3..9b8e898c 100755 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- 中文  ｜  English + 中文  ｜  English  ｜  日本語

# SimAI @@ -11,13 +11,10 @@ ### Recent Updates -- [2026/03] **SimAI 1.6 Released!** This release adds GPU memory modeling for inference simulation. Key features include: - - - **GPU Memory Module:** Accurate parameter counting and KV cache management for DeepSeek-V3-671B, Qwen3-MoE-235B, and Qwen3-Next-80B. - - **PD-Separation Memory Planning:** Independent parameter memory and KV cache budget calculation for Prefill and Decode phases. See [memory_planner.py](./vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py). - - **Improved Decode Time Estimation:** Linear interpolation replacing nearest-neighbor for AICB decode time prediction, with global cache for cross-run reuse. See [execution_time.py](./vidur-alibabacloud/vidur/entities/execution_time.py). - - **4-Scenario Test Suite:** End-to-end validation covering Qwen3-Next-80B, DeepSeek-671B, and Qwen3-MoE-235B. See [run_scenarios.sh](./vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh). - - **Code Quality:** Replaced print with logging, added bilingual docstrings, removed ~390 lines of dead code, standardized TODOs, and added type annotations across vidur-alibabacloud modules. +- [2026/04] **SimAI 1.6 Released!** Key updates: + - GPU memory modeling for inference simulation (parameter counting & KV cache). + - Linear interpolation for decode time estimation (replacing nearest-neighbor). + - Prefill-Decode Disaggregation memory planning (independent budgets for Prefill/Decode). - [2025/12] **SimAI 1.5 Released!** This release brings end-to-end simulation for multi-request **inference** workloads. Key features include: @@ -47,7 +44,7 @@ | Date | Event | Location | Content | Type | |:----------------:|:------------------------------------------------------------------------ |:----------------------- |:-------------------------------------------------------- |:-------------:| -| Mar 16, 2026 | SimAI 1.6 | 🌐 Online | The release of SimAI 1.6 | 💻 Virtual | +| Apr 23, 2026 | SimAI 1.6 | 🌐 Online | The release of SimAI 1.6 | 💻 Virtual | | Dec 30, 2025 | SimAI 1.5 | 🌐 Online | The release of SimAI 1.5 | 💻 Virtual | | Jun 4, 2025 | The first workshop of the SimAI community | 📍 Peking University | Three talks from community contributors | 🎓 On-site | | May 24, 2025 | The 28th Chinasys workshop | 📍 Chongqing University | An invited talk about SimAI | 🎓 On-site | diff --git a/README_CN.md b/README_CN.md index 0baf73a3..357afc4d 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1,5 +1,5 @@

- 中文  ｜  English + 中文  ｜  English  ｜  日本語

# SimAI @@ -11,13 +11,10 @@ ### 近期更新 -- [2026/03] **SimAI 1.6 正式发布！** 本版本新增推理仿真的 GPU 内存建模能力。主要特性包括： - - - **GPU 内存计算模块：** 支持 DeepSeek-V3-671B、Qwen3-MoE-235B、Qwen3-Next-80B 的精确参数计数与 KV Cache 管理。 - - **PD 分离内存规划：** Prefill 与 Decode 阶段独立的参数内存和 KV Cache 预算计算。详见 [memory_planner.py](./vidur-alibabacloud/vidur/scheduler/utils/memory_planner.py)。 - - **Decode 时间估算改进：** 首尾线性插值替代最近邻的 AICB decode 时间预测，全局缓存支持跨运行复用。详见 [execution_time.py](./vidur-alibabacloud/vidur/entities/execution_time.py)。 - - **4 场景端到端测试：** 覆盖 Qwen3-Next-80B、DeepSeek-671B、Qwen3-MoE-235B 的完整验证套件。详见 [run_scenarios.sh](./vidur-alibabacloud/examples/vidur-ali-scenarios/run_scenarios.sh)。 - - **代码质量提升：** logging 替换 print 输出、双语 docstring、清理 ~390 行死代码、TODO 规范化、类型标注补全。 +- [2026/04] **SimAI 1.6 正式发布！** 主要更新： + - 推理仿真 GPU 显存建模（参数计数与 KV Cache 管理）。 + - Decode 耗时线性插值估算（替代最近邻查找）。 + - PD 分离内存规划（Prefill/Decode 独立预算）。 - [2025/12] **SimAI 1.5 正式发布！** 本版本新增对多请求**推理**工作负载的端到端仿真支持，主要特性包括： @@ -46,7 +43,7 @@ | 日期 | 活动 | 地点 | 内容 | 形式 | |:----------------:|:------------------------------------------------------------------------ |:----------------------- |:-------------------------------------------------------- |:-------------:| -| Mar 16, 2026 | SimAI 1.6 | 🌐 线上 | SimAI 1.6 正式发布 | 💻 线上直播 | +| Apr 23, 2026 | SimAI 1.6 | 🌐 线上 | SimAI 1.6 正式发布 | 💻 线上直播 | | Dec 30, 2025 | SimAI 1.5 | 🌐 线上 | SimAI 1.5 正式发布 | 💻 线上直播 | | Jun 4, 2025 | SimAI 社区第一届研讨会 | 📍 北京大学 | 三场社区贡献者演讲 | 🎓 线下 | | May 24, 2025 | 第 28 届 Chinasys 研讨会 | 📍 重庆大学 | SimAI 受邀演讲 | 🎓 线下 | diff --git a/vidur-alibabacloud/README.md b/vidur-alibabacloud/README.md index df99cfcd..31bacf22 100644 --- a/vidur-alibabacloud/README.md +++ b/vidur-alibabacloud/README.md @@ -31,7 +31,7 @@ Vidur ([original](https://github.com/microsoft/vidur)) is a simulation framework ## Key Features -- **Prefill–Decode (PD) Separation** — Enables running the prefill and decode stages on different nodes, allowing elastic resource allocation and performance isolation. +- **Prefill–Decode (PD) Disaggregation** — Enables running the prefill and decode stages on different nodes, allowing elastic resource allocation and performance isolation. (Inspired by [splitwise-sim](https://github.com/Mutinifni/splitwise-sim)) - **Flexible Parallelism** — Supports: - **Data Parallel (DP)** @@ -332,7 +332,7 @@ The following scenarios are pre-configured in `run_scenarios.sh`. All scenarios - PD P2P bandwidth: 800 Gbps, dtype: fp8 - Request: Poisson QPS=100, 4 requests, fixed prefill=100 / decode=8 tokens -| Scenario | Model | PD Separation | World Size | TP | PP | EP | Global Scheduler | +| Scenario | Model | PD Disaggregation | World Size | TP | PP | EP | Global Scheduler | |----------|-------|---------------|------------|----|----|------------|------------------| | 1 | Qwen3-Next-80B (MoE) | No | 32 (dp=32) | 1 | 1 | auto (=world_size) | lor | | 2 | Qwen3-Next-80B (MoE) | Yes (P=2, D=6) | 8 | 1 | 1 | auto (=world_size) | split_wise | @@ -385,7 +385,7 @@ Each run produces the following directory: | `--trace_request_length_generator_config_trace_file` | `data/processed_traces/sharegpt_8k_filtered_stats_llama2_tokenizer.csv` | Path to trace file for request lengths | | `--interval_generator_config_type` | poisson | Inter-arrival time generator type | | `--cluster_config_num_replicas` | 1 | Total number of replicas (i.e., data parallelism degree) | -| `--replica_config_pd_node_ratio` | 1 | Fraction of replicas allocated as prefill (P) nodes. 1 = MIXED mode (no PD separation). (0, 1) = PD separation enabled. E.g., 0.5 means P:D = 1:1. | +| `--replica_config_pd_node_ratio` | 1 | Fraction of replicas allocated as prefill (P) nodes. 1 = MIXED mode (no PD disaggregation). (0, 1) = PD disaggregation enabled. E.g., 0.5 means P:D = 1:1. | | `--global_scheduler_config_type` | round_robin | Global scheduler type (`split_wise`, `round_robin`, etc.) | | `--replica_scheduler_config_type` | sarathi | Per-replica scheduler type | | `--replica_config_model_name` | meta-llama/Llama-2-7b-hf | Model name (DeepSeek-671B, Qwen3-MoE-235B, Qwen3-Next-80B, etc.) | @@ -397,7 +397,7 @@ Each run produces the following directory: | `--random_forrest_execution_time_predictor_config_simai_simulation_topo` | `../example/topo` | Path to SimAI topology file (only effective when backend = `simai_simulation`) | | `--random_forrest_execution_time_predictor_config_simai_simulation_config` | `../astra-sim-alibabacloud/inputs/config/SimAI.conf` | Path to SimAI configuration file (only effective when backend = `simai_simulation`) | -### PD Separation Parameters +### PD Disaggregation Parameters When `pd_node_ratio` < 1, the following optional parameters become effective: @@ -409,7 +409,7 @@ When `pd_node_ratio` < 1, the following optional parameters become effective: | `--replica_config_decode_num_pipeline_stages` | None | Decode-specific PP size. Falls back to `num_pipeline_stages` if not set. | | `--replica_config_num_prefill_replicas` | None | Directly specify prefill replica count (takes priority over `pd_node_ratio`). | -**Example: DeepSeek-671B with PD separation (P:D = 2:6)** +**Example: DeepSeek-671B with PD disaggregation (P:D = 2:6)** ```bash python -m vidur.main \ @@ -461,7 +461,7 @@ Simulation results are saved to: **Notes:** - All time-related fields are in **seconds (s)**, based on monotonic clock or Unix timestamps. -- In non-PD-separated deployments, `prefill_replica_id` and `decode_replica_id` are typically identical. +- In non-PD-disaggregated deployments, `prefill_replica_id` and `decode_replica_id` are typically identical. - If `request_num_decode_tokens = 0`, `tbt` is undefined (may be recorded as `NaN` or `0`). - TBT is not yet logged in `request_metrics.csv`; it can be computed manually for now. From d17c6a14ef515439aff1e2f88a7a2c30cbde6119 Mon Sep 17 00:00:00 2001 From: tianhao909 <843101550@qq.com> Date: Thu, 23 Apr 2026 10:04:00 +0000 Subject: [PATCH 8/8] docs(vidur-alibabacloud): restore README-vidur.md to upstream + migrate scenario commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [EN] - Restore README-vidur.md to match official Microsoft Vidur README (remove SimAI extensions) - Migrate 4-scenario manual CLI commands to README.md and README_CN.md - Add detailed per-scenario python commands after "Run 4-Scenario Suite" section - Keep run_scenarios.sh quick-start + full manual commands for advanced users [ZH] - 恢复 README-vidur.md 与官方 Microsoft Vidur README 完全一致（移除 SimAI 扩展内容） - 将四场景手动 CLI 命令迁移至 README.md 和 README_CN.md - 在"运行四场景套件"小节后添加逐场景完整 python 命令 - 保留脚本一键运行 + 完整手动命令两种方式供用户选择 --- vidur-alibabacloud/README-vidur.md | 157 ----------------------------- vidur-alibabacloud/README.md | 139 +++++++++++++++++++++++++ vidur-alibabacloud/README_CN.md | 141 ++++++++++++++++++++++++++ 3 files changed, 280 insertions(+), 157 deletions(-) diff --git a/vidur-alibabacloud/README-vidur.md b/vidur-alibabacloud/README-vidur.md index 786ea177..d0def19e 100644 --- a/vidur-alibabacloud/README-vidur.md +++ b/vidur-alibabacloud/README-vidur.md @@ -166,160 +166,3 @@ trademarks or logos is subject to and must follow Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party's policies. -## SimAI / AICB 场景示例（一键运行） - -> 以下命令均在 `vidur-alibabacloud/` 目录下执行，需提前激活 `vidur` conda 环境。 -> 使用 AICB 后端 (`--random_forrest_execution_time_predictor_config_backend aicb`)， -> 设备为 H20 DGX (`h20_dgx`)，请求生成为 Poisson QPS=100，固定长度 prefill=100/decode=8。 -> 所有输入输出文件统一汇聚至 `examples/vidur-ali-scenarios/` 目录： -> - 脚本: `examples/vidur-ali-scenarios/run_scenarios.sh` -> - 运行日志: `examples/vidur-ali-scenarios/logs/scenario__.log` -> - 模拟输出: `examples/vidur-ali-scenarios/simulator_output//` - -### 场景汇总 - -| 场景 | 模型 | ws | TP | PP | EP | PD分离 | 调度器 | -|------|------|----|----|----|-----|--------|--------| -| 1 | Qwen3-Next-80B | 32 | 1 | 1 | auto (=ws) | 否 | lor | -| 2 | Qwen3-Next-80B | 8 (P=2,D=6) | 1 | 1 | auto (=ws) | 是 | split_wise | -| 3 | DeepSeek-671B | 8 (P=2,D=6) | 8 | 1 | auto (=ws) | 是 | split_wise | -| 4 | Qwen3-MoE-235B | 8 (P=2,D=6) | 4 | 1 | auto (=ws) | 是 | split_wise | - -### 使用方法 - -```sh -# 激活环境 -conda activate vidur - -# 运行单个场景（1~4） -bash examples/vidur-ali-scenarios/run_scenarios.sh --scenario 1 - -# 顺序运行所有场景 -bash examples/vidur-ali-scenarios/run_scenarios.sh --all - -# 查看帮助 -bash examples/vidur-ali-scenarios/run_scenarios.sh --help -``` - -### 分场景命令（手动方式） - -**场景 1: Qwen3-Next-80B 无PD分离 (ws=32, lor)** - -```sh -python -m vidur.main \ - --replica_config_pd_p2p_comm_bandwidth 800 \ - --replica_config_nvlink_bandwidth 1600 \ - --replica_config_rdma_bandwidth 800 \ - --replica_config_pd_p2p_comm_dtype fp8 \ - --replica_config_network_device h20_dgx \ - --replica_config_device h20 \ - --request_generator_config_type synthetic \ - --interval_generator_config_type poisson \ - --poisson_request_interval_generator_config_qps 100 \ - --synthetic_request_generator_config_num_requests 4 \ - --length_generator_config_type fixed \ - --fixed_request_length_generator_config_prefill_tokens 100 \ - --fixed_request_length_generator_config_decode_tokens 8 \ - --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ - --random_forrest_execution_time_predictor_config_backend aicb \ - --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ - --cluster_config_num_replicas 32 \ - --replica_config_pd_node_ratio 1 \ - --global_scheduler_config_type lor \ - --replica_scheduler_config_type sarathi \ - --replica_config_model_name qwen3-next-80B \ - --replica_config_tensor_parallel_size 1 \ - --replica_config_num_pipeline_stages 1 -``` - -**场景 2: Qwen3-Next-80B PD分离 (P=2, D=6, split_wise)** - -```sh -python -m vidur.main \ - --replica_config_pd_p2p_comm_bandwidth 800 \ - --replica_config_nvlink_bandwidth 1600 \ - --replica_config_rdma_bandwidth 800 \ - --replica_config_pd_p2p_comm_dtype fp8 \ - --replica_config_network_device h20_dgx \ - --replica_config_device h20 \ - --request_generator_config_type synthetic \ - --interval_generator_config_type poisson \ - --poisson_request_interval_generator_config_qps 100 \ - --synthetic_request_generator_config_num_requests 4 \ - --length_generator_config_type fixed \ - --fixed_request_length_generator_config_prefill_tokens 100 \ - --fixed_request_length_generator_config_decode_tokens 8 \ - --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ - --random_forrest_execution_time_predictor_config_backend aicb \ - --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ - --cluster_config_num_replicas 8 \ - --replica_config_pd_node_ratio 0.25 \ - --replica_config_num_prefill_replicas 2 \ - --global_scheduler_config_type split_wise \ - --replica_scheduler_config_type split_wise \ - --replica_config_model_name qwen3-next-80B \ - --replica_config_tensor_parallel_size 1 \ - --replica_config_num_pipeline_stages 1 \ - --replica_config_prefill_tensor_parallel_size 1 \ - --replica_config_prefill_num_pipeline_stages 1 \ - --replica_config_decode_tensor_parallel_size 1 \ - --replica_config_decode_num_pipeline_stages 1 -``` - -**场景 3: DeepSeek-671B PD分离 (tp=8, EP=auto, split_wise)** - -```sh -python -m vidur.main \ - --replica_config_pd_p2p_comm_bandwidth 800 \ - --replica_config_nvlink_bandwidth 1600 \ - --replica_config_rdma_bandwidth 800 \ - --replica_config_pd_p2p_comm_dtype fp8 \ - --replica_config_network_device h20_dgx \ - --replica_config_device h20 \ - --request_generator_config_type synthetic \ - --interval_generator_config_type poisson \ - --poisson_request_interval_generator_config_qps 100 \ - --synthetic_request_generator_config_num_requests 4 \ - --length_generator_config_type fixed \ - --fixed_request_length_generator_config_prefill_tokens 100 \ - --fixed_request_length_generator_config_decode_tokens 8 \ - --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ - --random_forrest_execution_time_predictor_config_backend aicb \ - --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ - --cluster_config_num_replicas 8 \ - --replica_config_pd_node_ratio 0.25 \ - --global_scheduler_config_type split_wise \ - --replica_scheduler_config_type split_wise \ - --replica_config_model_name deepseek-671B \ - --replica_config_tensor_parallel_size 8 \ - --replica_config_num_pipeline_stages 1 -``` - -**场景 4: Qwen3-MoE-235B PD分离 (tp=4, EP=auto, split_wise)** - -```sh -python -m vidur.main \ - --replica_config_pd_p2p_comm_bandwidth 800 \ - --replica_config_nvlink_bandwidth 1600 \ - --replica_config_rdma_bandwidth 800 \ - --replica_config_pd_p2p_comm_dtype fp8 \ - --replica_config_network_device h20_dgx \ - --replica_config_device h20 \ - --request_generator_config_type synthetic \ - --interval_generator_config_type poisson \ - --poisson_request_interval_generator_config_qps 100 \ - --synthetic_request_generator_config_num_requests 4 \ - --length_generator_config_type fixed \ - --fixed_request_length_generator_config_prefill_tokens 100 \ - --fixed_request_length_generator_config_decode_tokens 8 \ - --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ - --random_forrest_execution_time_predictor_config_backend aicb \ - --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ - --cluster_config_num_replicas 8 \ - --replica_config_pd_node_ratio 0.25 \ - --global_scheduler_config_type split_wise \ - --replica_scheduler_config_type split_wise \ - --replica_config_model_name qwen3-moe-235B \ - --replica_config_tensor_parallel_size 4 \ - --replica_config_num_pipeline_stages 1 -``` \ No newline at end of file diff --git a/vidur-alibabacloud/README.md b/vidur-alibabacloud/README.md index 31bacf22..624ad7a3 100644 --- a/vidur-alibabacloud/README.md +++ b/vidur-alibabacloud/README.md @@ -341,6 +341,145 @@ The following scenarios are pre-configured in `run_scenarios.sh`. All scenarios > **Note:** All four models use Mixture-of-Experts (MoE) architecture. EP is automatically set to the cluster world_size at runtime and cannot be manually overridden. +#### Usage + +```bash +# Activate environment +conda activate vidur + +# Run a single scenario (1~4) +bash examples/vidur-ali-scenarios/run_scenarios.sh --scenario 1 + +# Run all scenarios sequentially +bash examples/vidur-ali-scenarios/run_scenarios.sh --all + +# Show help +bash examples/vidur-ali-scenarios/run_scenarios.sh --help +``` + +#### Manual Commands (Per Scenario) + +**Scenario 1: Qwen3-Next-80B without PD Disaggregation (ws=32, lor)** + +```bash +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 32 \ + --replica_config_pd_node_ratio 1 \ + --global_scheduler_config_type lor \ + --replica_scheduler_config_type sarathi \ + --replica_config_model_name qwen3-next-80B \ + --replica_config_tensor_parallel_size 1 \ + --replica_config_num_pipeline_stages 1 +``` + +**Scenario 2: Qwen3-Next-80B with PD Disaggregation (P=2, D=6, split_wise)** + +```bash +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --replica_config_num_prefill_replicas 2 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name qwen3-next-80B \ + --replica_config_tensor_parallel_size 1 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_prefill_tensor_parallel_size 1 \ + --replica_config_prefill_num_pipeline_stages 1 \ + --replica_config_decode_tensor_parallel_size 1 \ + --replica_config_decode_num_pipeline_stages 1 +``` + +**Scenario 3: DeepSeek-671B with PD Disaggregation (tp=8, EP=auto, split_wise)** + +```bash +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name deepseek-671B \ + --replica_config_tensor_parallel_size 8 \ + --replica_config_num_pipeline_stages 1 +``` + +**Scenario 4: Qwen3-MoE-235B with PD Disaggregation (tp=4, EP=auto, split_wise)** + +```bash +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name qwen3-moe-235B \ + --replica_config_tensor_parallel_size 4 \ + --replica_config_num_pipeline_stages 1 +``` + #### Output Files **Output path depends on how you run the simulation:** diff --git a/vidur-alibabacloud/README_CN.md b/vidur-alibabacloud/README_CN.md index 874a794f..ca57cd51 100644 --- a/vidur-alibabacloud/README_CN.md +++ b/vidur-alibabacloud/README_CN.md @@ -340,6 +340,147 @@ bash examples/vidur-ali-scenarios/run_scenarios.sh --all > **说明：** 四个模型均使用混合专家（MoE）架构。EP 在运行时自动设为 cluster world_size，不支持手动指定。 +#### run_scenarios.sh 使用方法 + +```bash +# 激活环境 +conda activate vidur + +# 运行单个场景（1~4） +bash examples/vidur-ali-scenarios/run_scenarios.sh --scenario 1 + +# 顺序运行所有场景 +bash examples/vidur-ali-scenarios/run_scenarios.sh --all + +# 查看帮助 +bash examples/vidur-ali-scenarios/run_scenarios.sh --help +``` + +#### 手动运行命令（逐场景） + +以下为四个场景的完整 CLI 命令，可直接复制运行。所有命令均在 `vidur-alibabacloud/` 目录下执行。 + +**场景 1：Qwen3-Next-80B 无PD分离（ws=32, lor）** + +```bash +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 32 \ + --replica_config_pd_node_ratio 1 \ + --global_scheduler_config_type lor \ + --replica_scheduler_config_type sarathi \ + --replica_config_model_name qwen3-next-80B \ + --replica_config_tensor_parallel_size 1 \ + --replica_config_num_pipeline_stages 1 +``` + +**场景 2：Qwen3-Next-80B PD分离（P=2, D=6, split_wise）** + +```bash +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --replica_config_num_prefill_replicas 2 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name qwen3-next-80B \ + --replica_config_tensor_parallel_size 1 \ + --replica_config_num_pipeline_stages 1 \ + --replica_config_prefill_tensor_parallel_size 1 \ + --replica_config_prefill_num_pipeline_stages 1 \ + --replica_config_decode_tensor_parallel_size 1 \ + --replica_config_decode_num_pipeline_stages 1 +``` + +**场景 3：DeepSeek-671B PD分离（tp=8, EP=auto, split_wise）** + +```bash +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name deepseek-671B \ + --replica_config_tensor_parallel_size 8 \ + --replica_config_num_pipeline_stages 1 +``` + +**场景 4：Qwen3-MoE-235B PD分离（tp=4, EP=auto, split_wise）** + +```bash +python -m vidur.main \ + --replica_config_pd_p2p_comm_bandwidth 800 \ + --replica_config_nvlink_bandwidth 1600 \ + --replica_config_rdma_bandwidth 800 \ + --replica_config_pd_p2p_comm_dtype fp8 \ + --replica_config_network_device h20_dgx \ + --replica_config_device h20 \ + --request_generator_config_type synthetic \ + --interval_generator_config_type poisson \ + --poisson_request_interval_generator_config_qps 100 \ + --synthetic_request_generator_config_num_requests 4 \ + --length_generator_config_type fixed \ + --fixed_request_length_generator_config_prefill_tokens 100 \ + --fixed_request_length_generator_config_decode_tokens 8 \ + --trace_request_length_generator_config_trace_file ./data/processed_traces/splitwise_conv.csv \ + --random_forrest_execution_time_predictor_config_backend aicb \ + --metrics_config_output_dir examples/vidur-ali-scenarios/simulator_output \ + --cluster_config_num_replicas 8 \ + --replica_config_pd_node_ratio 0.25 \ + --global_scheduler_config_type split_wise \ + --replica_scheduler_config_type split_wise \ + --replica_config_model_name qwen3-moe-235B \ + --replica_config_tensor_parallel_size 4 \ + --replica_config_num_pipeline_stages 1 +``` + #### 输出文件说明 **输出路径取决于运行方式：**