diff --git a/.github/workflows/full-sweep-1k1k-scheduler.yml b/.github/workflows/full-sweep-1k1k-scheduler.yml deleted file mode 100644 index a8b40214e..000000000 --- a/.github/workflows/full-sweep-1k1k-scheduler.yml +++ /dev/null @@ -1,228 +0,0 @@ -name: "Full Sweep Scheduler - 1k1k" - -on: - workflow_dispatch: - -jobs: - get-dsr1-configs: - runs-on: ubuntu-latest - outputs: - multi-node-search-space-config: ${{ steps.get-dsr1-configs.outputs.multi-node-search-space-config }} - single-node-search-space-config: ${{ steps.get-dsr1-configs.outputs.single-node-search-space-config }} - steps: - - name: Checkout code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - id: get-dsr1-configs - run: | - pip install pydantic - CONFIG_JSON_MULTI_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --multi-node --seq-lens 1k1k --model-prefix dsr1 --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - CONFIG_JSON_SINGLE_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --single-node --seq-lens 1k1k --model-prefix dsr1 --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - echo "multi-node-search-space-config=$CONFIG_JSON_MULTI_NODE" >> $GITHUB_OUTPUT - echo "single-node-search-space-config=$CONFIG_JSON_SINGLE_NODE" >> $GITHUB_OUTPUT - - get-gptoss-configs: - runs-on: ubuntu-latest - outputs: - multi-node-search-space-config: ${{ steps.get-gptoss-configs.outputs.multi-node-search-space-config }} - single-node-search-space-config: ${{ steps.get-gptoss-configs.outputs.single-node-search-space-config }} - steps: - - name: Checkout code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - id: get-gptoss-configs - run: | - pip install pydantic - CONFIG_JSON_MULTI_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --multi-node --seq-lens 1k1k --model-prefix gptoss --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - CONFIG_JSON_SINGLE_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --single-node --seq-lens 1k1k --model-prefix gptoss --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - echo "multi-node-search-space-config=$CONFIG_JSON_MULTI_NODE" >> $GITHUB_OUTPUT - echo "single-node-search-space-config=$CONFIG_JSON_SINGLE_NODE" >> $GITHUB_OUTPUT - - benchmark-dsr1-multi-node: - needs: get-dsr1-configs - if: ${{ needs.get-dsr1-configs.outputs.multi-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-multinode-tmpl.yml - name: dsr1 1k1k multi-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-dsr1-configs.outputs.multi-node-search-space-config) }} - secrets: inherit - with: - exp-name: "dsr1_1k1k" - isl: 1024 - osl: 1024 - max-model-len: 2248 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - conc-list: ${{ toJson(matrix.config.conc) }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - prefill-num-worker: ${{ matrix.config.prefill.num-worker }} - prefill-tp: ${{ matrix.config.prefill.tp }} - prefill-ep: ${{ matrix.config.prefill.ep }} - prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} - prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} - - decode-num-worker: ${{ matrix.config.decode.num-worker }} - decode-tp: ${{ matrix.config.decode.tp }} - decode-ep: ${{ matrix.config.decode.ep }} - decode-dp-attn: ${{ matrix.config.decode.dp-attn }} - decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - - benchmark-dsr1-single-node: - needs: get-dsr1-configs - if: ${{ needs.get-dsr1-configs.outputs.single-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-tmpl.yml - name: dsr1 1k1k single-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-dsr1-configs.outputs.single-node-search-space-config) }} - secrets: inherit - with: - exp-name: "dsr1_1k1k" - isl: 1024 - osl: 1024 - max-model-len: 2248 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - tp: ${{ matrix.config.tp }} - ep: ${{ matrix.config.ep }} - dp-attn: ${{ matrix.config.dp-attn }} - conc: ${{ matrix.config.conc }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - benchmark-gptoss-multi-node: - needs: get-gptoss-configs - if: ${{ needs.get-gptoss-configs.outputs.multi-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-multinode-tmpl.yml - name: gptoss 1k1k multi-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-gptoss-configs.outputs.multi-node-search-space-config) }} - secrets: inherit - with: - exp-name: "gptoss_1k1k" - isl: 1024 - osl: 1024 - max-model-len: 2248 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - conc-list: ${{ toJson(matrix.config.conc) }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - prefill-num-worker: ${{ matrix.config.prefill.num-worker }} - prefill-tp: ${{ matrix.config.prefill.tp }} - prefill-ep: ${{ matrix.config.prefill.ep }} - prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} - prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} - - decode-num-worker: ${{ matrix.config.decode.num-worker }} - decode-tp: ${{ matrix.config.decode.tp }} - decode-ep: ${{ matrix.config.decode.ep }} - decode-dp-attn: ${{ matrix.config.decode.dp-attn }} - decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - - benchmark-gptoss-single-node: - needs: get-gptoss-configs - if: ${{ needs.get-gptoss-configs.outputs.single-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-tmpl.yml - name: gptoss 1k1k single-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-gptoss-configs.outputs.single-node-search-space-config) }} - secrets: inherit - with: - exp-name: "gptoss_1k1k" - isl: 1024 - osl: 1024 - max-model-len: 2248 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - tp: ${{ matrix.config.tp }} - ep: ${{ matrix.config.ep }} - dp-attn: ${{ matrix.config.dp-attn }} - conc: ${{ matrix.config.conc }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - collect-dsr1-results: - needs: - [ - get-dsr1-configs, - benchmark-dsr1-single-node, - benchmark-dsr1-multi-node, - ] - if: ${{ always() && needs.get-dsr1-configs.result == 'success' }} - uses: ./.github/workflows/collect-results.yml - secrets: inherit - with: - exp-name: "dsr1_1k1k" - - collect-gptoss-results: - needs: - [ - get-gptoss-configs, - benchmark-gptoss-single-node, - benchmark-gptoss-multi-node, - ] - if: ${{ always() && needs.get-gptoss-configs.result == 'success' }} - uses: ./.github/workflows/collect-results.yml - secrets: inherit - with: - exp-name: "gptoss_1k1k" - - calc-success-rate: - needs: [collect-dsr1-results, collect-gptoss-results] - if: ${{ always() }} - runs-on: ubuntu-latest - - env: - RESULTS_DIR: "results/" - STATS_FILENAME: "run_stats" - GITHUB_TOKEN: ${{ secrets.REPO_PAT }} - - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - token: ${{ secrets.REPO_PAT }} - fetch-depth: 0 - - - name: Download results artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - path: ${{ env.RESULTS_DIR }} - pattern: results_* - - - name: Install python dependencies - run: pip install PyGithub - - - name: Calculate success rate - run: python3 utils/calc_success_rate.py $STATS_FILENAME - - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: "run-stats" - path: ${{ env.STATS_FILENAME }}.json diff --git a/.github/workflows/full-sweep-1k8k-scheduler.yml b/.github/workflows/full-sweep-1k8k-scheduler.yml deleted file mode 100644 index 062f00265..000000000 --- a/.github/workflows/full-sweep-1k8k-scheduler.yml +++ /dev/null @@ -1,228 +0,0 @@ -name: "Full Sweep Scheduler - 1k8k" - -on: - workflow_dispatch: - -jobs: - get-dsr1-configs: - runs-on: ubuntu-latest - outputs: - multi-node-search-space-config: ${{ steps.get-dsr1-configs.outputs.multi-node-search-space-config }} - single-node-search-space-config: ${{ steps.get-dsr1-configs.outputs.single-node-search-space-config }} - steps: - - name: Checkout code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - id: get-dsr1-configs - run: | - pip install pydantic - CONFIG_JSON_MULTI_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --multi-node --seq-lens 1k8k --model-prefix dsr1 --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - CONFIG_JSON_SINGLE_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --single-node --seq-lens 1k8k --model-prefix dsr1 --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - echo "multi-node-search-space-config=$CONFIG_JSON_MULTI_NODE" >> $GITHUB_OUTPUT - echo "single-node-search-space-config=$CONFIG_JSON_SINGLE_NODE" >> $GITHUB_OUTPUT - - get-gptoss-configs: - runs-on: ubuntu-latest - outputs: - multi-node-search-space-config: ${{ steps.get-gptoss-configs.outputs.multi-node-search-space-config }} - single-node-search-space-config: ${{ steps.get-gptoss-configs.outputs.single-node-search-space-config }} - steps: - - name: Checkout code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - id: get-gptoss-configs - run: | - pip install pydantic - CONFIG_JSON_MULTI_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --multi-node --seq-lens 1k8k --model-prefix gptoss --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - CONFIG_JSON_SINGLE_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --single-node --seq-lens 1k8k --model-prefix gptoss --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - echo "multi-node-search-space-config=$CONFIG_JSON_MULTI_NODE" >> $GITHUB_OUTPUT - echo "single-node-search-space-config=$CONFIG_JSON_SINGLE_NODE" >> $GITHUB_OUTPUT - - benchmark-dsr1-multi-node: - needs: get-dsr1-configs - if: ${{ needs.get-dsr1-configs.outputs.multi-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-multinode-tmpl.yml - name: dsr1 1k8k multi-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-dsr1-configs.outputs.multi-node-search-space-config) }} - secrets: inherit - with: - exp-name: "dsr1_1k8k" - isl: 1024 - osl: 8192 - max-model-len: 9416 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - conc-list: ${{ toJson(matrix.config.conc) }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - prefill-num-worker: ${{ matrix.config.prefill.num-worker }} - prefill-tp: ${{ matrix.config.prefill.tp }} - prefill-ep: ${{ matrix.config.prefill.ep }} - prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} - prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} - - decode-num-worker: ${{ matrix.config.decode.num-worker }} - decode-tp: ${{ matrix.config.decode.tp }} - decode-ep: ${{ matrix.config.decode.ep }} - decode-dp-attn: ${{ matrix.config.decode.dp-attn }} - decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - - benchmark-dsr1-single-node: - needs: get-dsr1-configs - if: ${{ needs.get-dsr1-configs.outputs.single-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-tmpl.yml - name: dsr1 1k8k single-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-dsr1-configs.outputs.single-node-search-space-config) }} - secrets: inherit - with: - exp-name: "dsr1_1k8k" - isl: 1024 - osl: 8192 - max-model-len: 9416 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - tp: ${{ matrix.config.tp }} - ep: ${{ matrix.config.ep }} - dp-attn: ${{ matrix.config.dp-attn }} - conc: ${{ matrix.config.conc }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - benchmark-gptoss-multi-node: - needs: get-gptoss-configs - if: ${{ needs.get-gptoss-configs.outputs.multi-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-multinode-tmpl.yml - name: gptoss 1k8k multi-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-gptoss-configs.outputs.multi-node-search-space-config) }} - secrets: inherit - with: - exp-name: "gptoss_1k8k" - isl: 1024 - osl: 8192 - max-model-len: 9416 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - conc-list: ${{ toJson(matrix.config.conc) }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - prefill-num-worker: ${{ matrix.config.prefill.num-worker }} - prefill-tp: ${{ matrix.config.prefill.tp }} - prefill-ep: ${{ matrix.config.prefill.ep }} - prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} - prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} - - decode-num-worker: ${{ matrix.config.decode.num-worker }} - decode-tp: ${{ matrix.config.decode.tp }} - decode-ep: ${{ matrix.config.decode.ep }} - decode-dp-attn: ${{ matrix.config.decode.dp-attn }} - decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - - benchmark-gptoss-single-node: - needs: get-gptoss-configs - if: ${{ needs.get-gptoss-configs.outputs.single-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-tmpl.yml - name: gptoss 1k8k single-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-gptoss-configs.outputs.single-node-search-space-config) }} - secrets: inherit - with: - exp-name: "gptoss_1k8k" - isl: 1024 - osl: 8192 - max-model-len: 9416 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - tp: ${{ matrix.config.tp }} - ep: ${{ matrix.config.ep }} - dp-attn: ${{ matrix.config.dp-attn }} - conc: ${{ matrix.config.conc }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - collect-dsr1-results: - needs: - [ - get-dsr1-configs, - benchmark-dsr1-single-node, - benchmark-dsr1-multi-node, - ] - if: ${{ always() && needs.get-dsr1-configs.result == 'success' }} - uses: ./.github/workflows/collect-results.yml - secrets: inherit - with: - exp-name: "dsr1_1k8k" - - collect-gptoss-results: - needs: - [ - get-gptoss-configs, - benchmark-gptoss-single-node, - benchmark-gptoss-multi-node, - ] - if: ${{ always() && needs.get-gptoss-configs.result == 'success' }} - uses: ./.github/workflows/collect-results.yml - secrets: inherit - with: - exp-name: "gptoss_1k8k" - - calc-success-rate: - needs: [collect-dsr1-results, collect-gptoss-results] - if: ${{ always() }} - runs-on: ubuntu-latest - - env: - RESULTS_DIR: "results/" - STATS_FILENAME: "run_stats" - GITHUB_TOKEN: ${{ secrets.REPO_PAT }} - - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - token: ${{ secrets.REPO_PAT }} - fetch-depth: 0 - - - name: Download results artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - path: ${{ env.RESULTS_DIR }} - pattern: results_* - - - name: Install python dependencies - run: pip install PyGithub - - - name: Calculate success rate - run: python3 utils/calc_success_rate.py $STATS_FILENAME - - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: "run-stats" - path: ${{ env.STATS_FILENAME }}.json diff --git a/.github/workflows/full-sweep-8k1k-scheduler.yml b/.github/workflows/full-sweep-8k1k-scheduler.yml deleted file mode 100644 index 2b45b9679..000000000 --- a/.github/workflows/full-sweep-8k1k-scheduler.yml +++ /dev/null @@ -1,228 +0,0 @@ -name: "Full Sweep Scheduler - 8k1k" - -on: - workflow_dispatch: - -jobs: - get-dsr1-configs: - runs-on: ubuntu-latest - outputs: - multi-node-search-space-config: ${{ steps.get-dsr1-configs.outputs.multi-node-search-space-config }} - single-node-search-space-config: ${{ steps.get-dsr1-configs.outputs.single-node-search-space-config }} - steps: - - name: Checkout code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - id: get-dsr1-configs - run: | - pip install pydantic - CONFIG_JSON_MULTI_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --multi-node --seq-lens 8k1k --model-prefix dsr1 --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - CONFIG_JSON_SINGLE_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --single-node --seq-lens 8k1k --model-prefix dsr1 --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - echo "multi-node-search-space-config=$CONFIG_JSON_MULTI_NODE" >> $GITHUB_OUTPUT - echo "single-node-search-space-config=$CONFIG_JSON_SINGLE_NODE" >> $GITHUB_OUTPUT - - get-gptoss-configs: - runs-on: ubuntu-latest - outputs: - multi-node-search-space-config: ${{ steps.get-gptoss-configs.outputs.multi-node-search-space-config }} - single-node-search-space-config: ${{ steps.get-gptoss-configs.outputs.single-node-search-space-config }} - steps: - - name: Checkout code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - id: get-gptoss-configs - run: | - pip install pydantic - CONFIG_JSON_MULTI_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --multi-node --seq-lens 8k1k --model-prefix gptoss --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - CONFIG_JSON_SINGLE_NODE=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py full-sweep --single-node --seq-lens 8k1k --model-prefix gptoss --config-files ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml) - echo "multi-node-search-space-config=$CONFIG_JSON_MULTI_NODE" >> $GITHUB_OUTPUT - echo "single-node-search-space-config=$CONFIG_JSON_SINGLE_NODE" >> $GITHUB_OUTPUT - - benchmark-dsr1-multi-node: - needs: get-dsr1-configs - if: ${{ needs.get-dsr1-configs.outputs.multi-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-multinode-tmpl.yml - name: dsr1 8k1k multi-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-dsr1-configs.outputs.multi-node-search-space-config) }} - secrets: inherit - with: - exp-name: "dsr1_8k1k" - isl: 8192 - osl: 1024 - max-model-len: 9416 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - conc-list: ${{ toJson(matrix.config.conc) }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - prefill-num-worker: ${{ matrix.config.prefill.num-worker }} - prefill-tp: ${{ matrix.config.prefill.tp }} - prefill-ep: ${{ matrix.config.prefill.ep }} - prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} - prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} - - decode-num-worker: ${{ matrix.config.decode.num-worker }} - decode-tp: ${{ matrix.config.decode.tp }} - decode-ep: ${{ matrix.config.decode.ep }} - decode-dp-attn: ${{ matrix.config.decode.dp-attn }} - decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - - benchmark-dsr1-single-node: - needs: get-dsr1-configs - if: ${{ needs.get-dsr1-configs.outputs.single-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-tmpl.yml - name: dsr1 8k1k single-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-dsr1-configs.outputs.single-node-search-space-config) }} - secrets: inherit - with: - exp-name: "dsr1_8k1k" - isl: 8192 - osl: 1024 - max-model-len: 9416 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - tp: ${{ matrix.config.tp }} - ep: ${{ matrix.config.ep }} - dp-attn: ${{ matrix.config.dp-attn }} - conc: ${{ matrix.config.conc }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - benchmark-gptoss-multi-node: - needs: get-gptoss-configs - if: ${{ needs.get-gptoss-configs.outputs.multi-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-multinode-tmpl.yml - name: gptoss 8k1k multi-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-gptoss-configs.outputs.multi-node-search-space-config) }} - secrets: inherit - with: - exp-name: "gptoss_8k1k" - isl: 8192 - osl: 1024 - max-model-len: 9416 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - conc-list: ${{ toJson(matrix.config.conc) }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - prefill-num-worker: ${{ matrix.config.prefill.num-worker }} - prefill-tp: ${{ matrix.config.prefill.tp }} - prefill-ep: ${{ matrix.config.prefill.ep }} - prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} - prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} - - decode-num-worker: ${{ matrix.config.decode.num-worker }} - decode-tp: ${{ matrix.config.decode.tp }} - decode-ep: ${{ matrix.config.decode.ep }} - decode-dp-attn: ${{ matrix.config.decode.dp-attn }} - decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - - benchmark-gptoss-single-node: - needs: get-gptoss-configs - if: ${{ needs.get-gptoss-configs.outputs.single-node-search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-tmpl.yml - name: gptoss 8k1k single-node / - strategy: - fail-fast: false - matrix: - config: ${{ fromJson(needs.get-gptoss-configs.outputs.single-node-search-space-config) }} - secrets: inherit - with: - exp-name: "gptoss_8k1k" - isl: 8192 - osl: 1024 - max-model-len: 9416 - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - tp: ${{ matrix.config.tp }} - ep: ${{ matrix.config.ep }} - dp-attn: ${{ matrix.config.dp-attn }} - conc: ${{ matrix.config.conc }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - collect-dsr1-results: - needs: - [ - get-dsr1-configs, - benchmark-dsr1-single-node, - benchmark-dsr1-multi-node, - ] - if: ${{ always() && needs.get-dsr1-configs.result == 'success' }} - uses: ./.github/workflows/collect-results.yml - secrets: inherit - with: - exp-name: "dsr1_8k1k" - - collect-gptoss-results: - needs: - [ - get-gptoss-configs, - benchmark-gptoss-single-node, - benchmark-gptoss-multi-node, - ] - if: ${{ always() && needs.get-gptoss-configs.result == 'success' }} - uses: ./.github/workflows/collect-results.yml - secrets: inherit - with: - exp-name: "gptoss_8k1k" - - calc-success-rate: - needs: [collect-dsr1-results, collect-gptoss-results] - if: ${{ always() }} - runs-on: ubuntu-latest - - env: - RESULTS_DIR: "results/" - STATS_FILENAME: "run_stats" - GITHUB_TOKEN: ${{ secrets.REPO_PAT }} - - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - token: ${{ secrets.REPO_PAT }} - fetch-depth: 0 - - - name: Download results artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - path: ${{ env.RESULTS_DIR }} - pattern: results_* - - - name: Install python dependencies - run: pip install PyGithub - - - name: Calculate success rate - run: python3 utils/calc_success_rate.py $STATS_FILENAME - - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: "run-stats" - path: ${{ env.STATS_FILENAME }}.json