diff --git a/.github/workflows/label-validation.yml b/.github/workflows/label-validation.yml deleted file mode 100644 index 1b01deb84..000000000 --- a/.github/workflows/label-validation.yml +++ /dev/null @@ -1,151 +0,0 @@ -name: PR Label Validation -run-name: "Validate PR #${{ github.event.pull_request.number }}" - -concurrency: - group: "PR#${{ github.event.pull_request.number }}" - cancel-in-progress: true - -on: - pull_request_target: - types: labeled - -jobs: - get-jobs: - runs-on: ubuntu-latest - environment: fork-pr-validation - outputs: - search-space-config: ${{ steps.get-jobs.outputs.search-space-config }} - steps: - - name: Checkout code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - id: get-jobs - shell: python - run: | - import json - import subprocess - import re - import os - - # Get matching labels - labels = json.loads(r'''${{ toJson(github.event.pull_request.labels) }}''') - pattern = r'^([^_]+)_([^_]+)$' - - matching = [] - for label in labels: - match = re.match(pattern, label['name']) - if match: - runner_type = match.group(1) - model_prefix = match.group(2) - - matching.append({'runner-type': runner_type, 'model-prefix': model_prefix}) - print(f"Matched label: {label['name']}") - - if not matching: - print("No matching labels found") - with open(os.environ['GITHUB_OUTPUT'], 'a') as f: - f.write('search-space-config=[]\n') - exit(0) - - # Generate configs for standard labels - all_configs = [] - if matching: - subprocess.run(['pip', 'install', 'pydantic'], check=True) - - for label in matching: - result = subprocess.run([ - 'python3', f"{os.environ['GITHUB_WORKSPACE']}/utils/matrix_logic/generate_sweep_configs.py", - 'full-sweep', - '--runner-type', label['runner-type'], - '--model-prefix', label['model-prefix'], - '--seq-lens', '1k1k', - '--config-files', - f"{os.environ['GITHUB_WORKSPACE']}/.github/configs/nvidia-master.yaml", - f"{os.environ['GITHUB_WORKSPACE']}/.github/configs/amd-master.yaml", - '--runner-config', f"{os.environ['GITHUB_WORKSPACE']}/.github/configs/runners.yaml" - ], capture_output=True, text=True) - - if result.returncode != 0: - print(f"Error generating configs:") - print(f"STDOUT: {result.stdout}") - print(f"STDERR: {result.stderr}") - exit(1) - - all_configs.extend(json.loads(result.stdout)) - - print(f"Total standard configs: {len(all_configs)}") - - with open(os.environ['GITHUB_OUTPUT'], 'a') as f: - f.write(f'search-space-config={json.dumps(all_configs)}\n') - - validate: - needs: get-jobs - if: ${{ always() && needs.get-jobs.result == 'success' && needs.get-jobs.outputs.search-space-config != '[]' }} - uses: ./.github/workflows/benchmark-tmpl.yml - strategy: - fail-fast: false - matrix: - # The '|| '[]'' fallback is required because GitHub evaluates the matrix expression - # before the 'if' condition. Without it, fromJson() throws a parse error when the - # output is empty/undefined (e.g., when get-jobs is pending approval or was skipped). - config: ${{ fromJson(needs.get-jobs.outputs.search-space-config || '[]') }} - secrets: inherit - name: validate ${{ matrix.config.runner }} - with: - exp-name: ${{ matrix.config.exp-name }} - isl: ${{ matrix.config.isl }} - osl: ${{ matrix.config.osl }} - max-model-len: ${{ matrix.config.max-model-len }} - runner: ${{ matrix.config.runner }} - image: ${{ matrix.config.image }} - model: ${{ matrix.config.model }} - model-prefix: ${{ matrix.config.model-prefix }} - framework: ${{ matrix.config.framework }} - precision: ${{ matrix.config.precision }} - tp: ${{ matrix.config.tp }} - ep: ${{ matrix.config.ep }} - dp-attn: ${{ matrix.config.dp-attn }} - conc: ${{ matrix.config.conc }} - spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: ${{ matrix.config.disagg }} - - collect-results: - needs: [get-jobs, validate] - if: ${{ always() && needs.get-jobs.result == 'success' }} - uses: ./.github/workflows/collect-results.yml - secrets: inherit - - calc-success-rate: - needs: [get-jobs, collect-results] - if: ${{ always() && needs.get-jobs.result == 'success' }} - runs-on: ubuntu-latest - - env: - RESULTS_DIR: "results/" - STATS_FILENAME: "run_stats" - GITHUB_TOKEN: ${{ secrets.REPO_PAT }} - - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - token: ${{ secrets.REPO_PAT }} - fetch-depth: 0 - - - name: Download results artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - path: ${{ env.RESULTS_DIR }} - pattern: results_* - - - name: Install python dependencies - run: pip install PyGithub - - - name: Calculate success rate - run: python3 utils/calc_success_rate.py $STATS_FILENAME - - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: "run-stats" - path: ${{ env.STATS_FILENAME }}.json