From f2ee3058985135011aa7d46acb9c307dde21fa2c Mon Sep 17 00:00:00 2001 From: ydshieh Date: Tue, 26 Aug 2025 10:36:18 +0200 Subject: [PATCH 1/6] up --- .github/workflows/push-important-models.yml | 260 ++++++++++++-------- .github/workflows/self-scheduled.yml | 9 +- .github/workflows/slack-report.yml | 2 + utils/notification_service.py | 12 +- utils/split_model_tests.py | 11 + utils/tests_fetcher.py | 31 +-- 6 files changed, 185 insertions(+), 140 deletions(-) diff --git a/.github/workflows/push-important-models.yml b/.github/workflows/push-important-models.yml index 099ded8018e9..bf71a2f129f0 100644 --- a/.github/workflows/push-important-models.yml +++ b/.github/workflows/push-important-models.yml @@ -2,7 +2,7 @@ name: Slow tests on important models (on Push - A10) on: push: - branches: [ main ] + branches: [ update_important_models ] env: OUTPUT_SLACK_CHANNEL_ID: "C06L2SGMEEA" @@ -25,111 +25,169 @@ jobs: - name: Check out code uses: actions/checkout@v4 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@1c8e6069583811afb28f97afeaf8e7da80c6be5c + - name: Get changed files using GitHub API + id: get-changed-files + uses: actions/github-script@v7 with: - files: src/transformers/models/** - - - name: Run step if only the files listed above change - if: steps.changed-files.outputs.any_changed == 'true' + script: | + let files = []; + + // Only handle push events + if (context.eventName === 'push') { + const afterSha = context.payload.after; + const branchName = context.payload.ref.replace('refs/heads/', ''); + + let baseSha; + + if (branchName === 'main') { + console.log('Push to main branch, comparing to parent commit'); + // Get the parent commit of the pushed commit + const { data: commit } = await github.rest.repos.getCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: afterSha + }); + baseSha = commit.parents[0]?.sha; + if (!baseSha) { + throw new Error('No parent commit found for the pushed commit'); + } + } else { + console.log(`Push to branch ${branchName}, comparing to main`); + baseSha = 'main'; + } + + const { data: comparison } = await github.rest.repos.compareCommits({ + owner: context.repo.owner, + repo: context.repo.repo, + base: baseSha, + head: afterSha + }); + + // Include added, modified, and renamed files + files = comparison.files + .filter(file => file.status === 'added' || file.status === 'modified' || file.status === 'renamed') + .map(file => file.filename); + } + + // Include all files under src/transformers/ (not just models subdirectory) + const filteredFiles = files.filter(file => + file.startsWith('src/transformers/') + ); + + core.setOutput('changed_files', filteredFiles.join(' ')); + core.setOutput('any_changed', filteredFiles.length > 0 ? 'true' : 'false'); + + - name: Parse changed files with Python + if: steps.get-changed-files.outputs.any_changed == 'true' id: set-matrix - env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} run: | - model_arrays=() - for file in $ALL_CHANGED_FILES; do - model_path="${file#*models/}" - model_path="models/${model_path%%/*}" - if grep -qFx "$model_path" utils/important_models.txt; then - # Append the file to the matrix string - model_arrays+=("$model_path") - fi - done - matrix_string=$(printf '"%s", ' "${model_arrays[@]}" | sed 's/, $//') - echo "matrix=[$matrix_string]" >> $GITHUB_OUTPUT - test_modified_files: + python3 - << 'EOF' + import os + import sys + import json + + # Add the utils directory to Python path + sys.path.insert(0, 'utils') + + # Import the important models list + from important_files import IMPORTANT_MODELS + + print(f"Important models: {IMPORTANT_MODELS}") + + # Get the changed files from the previous step + changed_files_str = "${{ steps.get-changed-files.outputs.changed_files }}" + changed_files = changed_files_str.split() if changed_files_str else [] + + # Filter to only Python files + python_files = [f for f in changed_files if f.endswith('.py')] + print(f"Python files changed: {python_files}") + + result_models = set() + + # Specific files that trigger all models + transformers_utils_files = [ + 'modeling_utils.py', + 'modeling_rope_utils.py', + 'modeling_flash_attention_utils.py', + 'modeling_attn_mask_utils.py', + 'cache_utils.py', + 'masking_utils.py', + 'pytorch_utils.py' + ] + + # Single loop through all Python files + for file in python_files: + # Check for files under src/transformers/models/ + if file.startswith('src/transformers/models/'): + remaining_path = file[len('src/transformers/models/'):] + if '/' in remaining_path: + model_dir = remaining_path.split('/')[0] + if model_dir in IMPORTANT_MODELS: + result_models.add(model_dir) + print(f"Added model directory: {model_dir}") + + # Check for specific files under src/transformers/ or generation files + elif file.startswith('src/transformers/generation/') or \ + (file.startswith('src/transformers/') and os.path.basename(file) in transformers_utils_files): + print(f"Found core file: {file} - including all important models") + result_models.update(IMPORTANT_MODELS) + break # No need to continue once we include all models + + # Convert to sorted list and create matrix + result_list = sorted(list(result_models)) + print(f"Final model list: {result_list}") + + if result_list: + matrix_json = json.dumps(result_list) + print(f"matrix={matrix_json}") + + # Write to GITHUB_OUTPUT + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f"matrix={matrix_json}\n") + else: + print("matrix=[]") + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write("matrix=[]\n") + EOF + + + test_matrix_output: + name: "Test matrix output" + runs-on: ubuntu-latest needs: get_modified_models - name: Slow & FA2 tests - runs-on: - group: aws-g5-4xlarge-cache - container: - image: huggingface/transformers-all-latest-gpu - options: --gpus all --privileged --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ - if: ${{ needs.get_modified_models.outputs.matrix != '[]' && needs.get_modified_models.outputs.matrix != '' && fromJson(needs.get_modified_models.outputs.matrix)[0] != null }} - strategy: - fail-fast: false - matrix: - model-name: ${{ fromJson(needs.get_modified_models.outputs.matrix) }} - + if: needs.get_modified_models.outputs.matrix != '[]' steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Install locally transformers & other libs - run: | - apt install sudo - sudo -H pip install --upgrade pip - sudo -H pip uninstall -y transformers - sudo -H pip install -U -e ".[testing]" - MAX_JOBS=4 pip install flash-attn --no-build-isolation - pip install bitsandbytes - - - name: NVIDIA-SMI + - name: Echo matrix value run: | - nvidia-smi - - - name: Show installed libraries and their versions - run: pip freeze - - - name: Run FA2 tests - id: run_fa2_tests - run: - pytest -rsfE -m "flash_attn_test" --make-reports=${{ matrix.model-name }}_fa2_tests/ tests/${{ matrix.model-name }}/test_modeling_* + echo "Full matrix was: ${{ needs.get_modified_models.outputs.matrix }}" - - name: "Test suite reports artifacts: ${{ matrix.model-name }}_fa2_tests" - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.model-name }}_fa2_tests - path: /transformers/reports/${{ matrix.model-name }}_fa2_tests - - - name: Post to Slack - if: always() - uses: huggingface/hf-workflows/.github/actions/post-slack@main - with: - slack_channel: ${{ env.OUTPUT_SLACK_CHANNEL_ID }} - title: 🤗 Results of the FA2 tests - ${{ matrix.model-name }} - status: ${{ steps.run_fa2_tests.conclusion}} - slack_token: ${{ secrets.CI_SLACK_BOT_TOKEN }} - - - name: Run integration tests - id: run_integration_tests - if: always() - run: - pytest -rsfE -k "IntegrationTest" --make-reports=tests_integration_${{ matrix.model-name }} tests/${{ matrix.model-name }}/test_modeling_* - - - name: "Test suite reports artifacts: tests_integration_${{ matrix.model-name }}" - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: tests_integration_${{ matrix.model-name }} - path: /transformers/reports/tests_integration_${{ matrix.model-name }} - - - name: Post to Slack - if: always() - uses: huggingface/hf-workflows/.github/actions/post-slack@main - with: - slack_channel: ${{ env.OUTPUT_SLACK_CHANNEL_ID }} - title: 🤗 Results of the Integration tests - ${{ matrix.model-name }} - status: ${{ steps.run_integration_tests.conclusion}} - slack_token: ${{ secrets.CI_SLACK_BOT_TOKEN }} - - name: Tailscale # In order to be able to SSH when a test fails - if: ${{ runner.debug == '1'}} - uses: huggingface/tailscale-action@v1 - with: - authkey: ${{ secrets.TAILSCALE_SSH_AUTHKEY }} - slackChannel: ${{ secrets.SLACK_CIFEEDBACK_CHANNEL }} - slackToken: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} - waitForSSH: true + model-ci: + name: Model CI + uses: ./.github/workflows/self-scheduled.yml + needs: get_modified_models + with: + job: run_models_gpu + slack_report_channel: "#transformers-ci-dummy" + docker: huggingface/transformers-all-latest-gpu + ci_event: push + report_repo_id: hf-internal-testing/transformers_ci_dummy + commit_sha: ${{ github.sha }} + models: ${{ needs.get_modified_models.outputs.matrix }} + secrets: inherit + + + +# test_matrix_output_with_matrix: +# name: "Test matrix output with matrix" +# runs-on: ubuntu-latest +# needs: get_modified_models +# if: needs.get_modified_models.outputs.matrix != '[]' +# strategy: +# matrix: +# model: ${{ fromJson(needs.get_modified_models.outputs.matrix) }} +# steps: +# - name: Echo matrix value +# run: | +# echo "Processing model: ${{ matrix.model }}" +# echo "Full matrix was: ${{ needs.get_modified_models.outputs.matrix }}" diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index b5f6685d30e7..cc44ee904aaa 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -31,7 +31,10 @@ on: commit_sha: required: false type: string - + models: + default: "" + required: false + type: string env: HF_HOME: /mnt/cache @@ -68,7 +71,7 @@ jobs: - name: Update clone working-directory: /transformers run: | - git fetch && git checkout ${{ github.sha }} + git fetch && git checkout ${{ inputs.commit_sha || github.sha }} - name: Cleanup working-directory: /transformers @@ -87,7 +90,7 @@ jobs: working-directory: /transformers/tests run: | if [ "${{ inputs.job }}" = "run_models_gpu" ]; then - echo "folder_slices=$(python3 ../utils/split_model_tests.py --num_splits ${{ env.NUM_SLICES }})" >> $GITHUB_OUTPUT + echo "folder_slices=$(python3 ../utils/split_model_tests.py --models '${{ inputs.models }}' --num_splits ${{ env.NUM_SLICES }})" >> $GITHUB_OUTPUT echo "slice_ids=$(python3 -c 'd = list(range(${{ env.NUM_SLICES }})); print(d)')" >> $GITHUB_OUTPUT echo "runner_map=$(python3 ../utils/get_runner_map.py)" >> $GITHUB_OUTPUT elif [ "${{ inputs.job }}" = "run_trainer_and_fsdp_gpu" ]; then diff --git a/.github/workflows/slack-report.yml b/.github/workflows/slack-report.yml index 88da4e38d4dc..4438e6e4de5b 100644 --- a/.github/workflows/slack-report.yml +++ b/.github/workflows/slack-report.yml @@ -75,6 +75,8 @@ jobs: SLACK_REPORT_CHANNEL: ${{ inputs.slack_report_channel }} ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }} CI_EVENT: ${{ inputs.ci_event }} + # This `CI_TITLE` would be empty for `schedule` or `workflow_run` events. + CI_TITLE: ${{ github.event.head_commit.message }} CI_SHA: ${{ inputs.commit_sha || github.sha }} CI_TEST_JOB: ${{ inputs.job }} SETUP_STATUS: ${{ inputs.setup_status }} diff --git a/utils/notification_service.py b/utils/notification_service.py index 9daceacb35af..410d3ba78507 100644 --- a/utils/notification_service.py +++ b/utils/notification_service.py @@ -1072,18 +1072,14 @@ def pop_default(l: list[Any], i: int, default: Any) -> Any: pr_number_re = re.compile(r"\(#(\d+)\)$") # Add Commit/PR title with a link for push CI - # (check the title in 2 env. variables - depending on the CI is triggered via `push` or `workflow_run` event) - ci_title_push = os.environ.get("CI_TITLE_PUSH") - ci_title_workflow_run = os.environ.get("CI_TITLE_WORKFLOW_RUN") - ci_title = ci_title_push if ci_title_push else ci_title_workflow_run - + ci_title = os.environ.get("CI_TITLE", "") ci_sha = os.environ.get("CI_SHA") ci_url = None if ci_sha: ci_url = f"https://github.com/{repository_full_name}/commit/{ci_sha}" - if ci_title is not None: + if ci_title: if ci_url is None: raise ValueError( "When a title is found (`ci_title`), it means a `push` event or a `workflow_run` even (triggered by " @@ -1112,9 +1108,9 @@ def pop_default(l: list[Any], i: int, default: Any) -> Any: merged_by = ci_details["merged_by"]["login"] if merged_by is None: - ci_title = f"<{ci_url}|{ci_title}>\nAuthor: {ci_author}" + ci_title = f"<{ci_url}|{ci_title}>\nAuthor: GH_{ci_author}" else: - ci_title = f"<{ci_url}|{ci_title}>\nAuthor: {ci_author} | Merged by: {merged_by}" + ci_title = f"<{ci_url}|{ci_title}>\nAuthor: GH_{ci_author} | Merged by: GH_{merged_by}" elif ci_sha: ci_title = f"<{ci_url}|commit: {ci_sha}>" diff --git a/utils/split_model_tests.py b/utils/split_model_tests.py index 3539a2fb3178..6a2aefb293a4 100644 --- a/utils/split_model_tests.py +++ b/utils/split_model_tests.py @@ -33,11 +33,18 @@ """ import argparse +import ast import os if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + type=str, + default="", + help="the list of pre-computed model names.", + ) parser.add_argument( "--num_splits", type=int, @@ -53,6 +60,10 @@ d1.remove("models") d = d2 + d1 + if args.models != "": + model_tests = ast.literal_eval(args.models) + d = sorted(filter(os.path.isdir, [f"models/{x}" for x in model_tests])) + num_jobs = len(d) num_jobs_per_splits = num_jobs // args.num_splits diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 7b8d98405ce9..bb188fee17f5 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -71,34 +71,9 @@ # This variable has effect only if `filter_models=False`. NUM_MODELS_TO_TRIGGER_FULL_CI = 30 -# List here the models to always test. -IMPORTANT_MODELS = [ - "auto", - "bert", - "gpt2", - "t5", - "modernbert", - "vit,clip", - "detr", - "table_transformer", - "got_ocr2", - "whisper", - "wav2vec2", - "qwen2_audio", - "speech_t5", - "csm", - "llama", - "gemma3", - "qwen2", - "mistral3", - "qwen2_5_vl", - "llava", - "smolvlm", - "internvl", - "gemma3n", - "gpt_oss", - "qwen2_5_omni", -] + +# List here the models not to be filtered by `filter_tests`. +from important_files import IMPORTANT_MODELS @contextmanager From 8ef268e68f7c8949f5e4e9644e420701bc107dfe Mon Sep 17 00:00:00 2001 From: ydshieh Date: Tue, 26 Aug 2025 11:16:24 +0200 Subject: [PATCH 2/6] up --- .github/workflows/push-important-models.yml | 32 ++------------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/.github/workflows/push-important-models.yml b/.github/workflows/push-important-models.yml index bf71a2f129f0..285f522d179c 100644 --- a/.github/workflows/push-important-models.yml +++ b/.github/workflows/push-important-models.yml @@ -150,44 +150,16 @@ jobs: f.write("matrix=[]\n") EOF - - test_matrix_output: - name: "Test matrix output" - runs-on: ubuntu-latest - needs: get_modified_models - if: needs.get_modified_models.outputs.matrix != '[]' - steps: - - name: Echo matrix value - run: | - echo "Full matrix was: ${{ needs.get_modified_models.outputs.matrix }}" - - model-ci: name: Model CI uses: ./.github/workflows/self-scheduled.yml needs: get_modified_models with: job: run_models_gpu - slack_report_channel: "#transformers-ci-dummy" + slack_report_channel: "#transformers-ci-push" docker: huggingface/transformers-all-latest-gpu ci_event: push - report_repo_id: hf-internal-testing/transformers_ci_dummy + report_repo_id: hf-internal-testing/transformers_ci_push commit_sha: ${{ github.sha }} models: ${{ needs.get_modified_models.outputs.matrix }} secrets: inherit - - - -# test_matrix_output_with_matrix: -# name: "Test matrix output with matrix" -# runs-on: ubuntu-latest -# needs: get_modified_models -# if: needs.get_modified_models.outputs.matrix != '[]' -# strategy: -# matrix: -# model: ${{ fromJson(needs.get_modified_models.outputs.matrix) }} -# steps: -# - name: Echo matrix value -# run: | -# echo "Processing model: ${{ matrix.model }}" -# echo "Full matrix was: ${{ needs.get_modified_models.outputs.matrix }}" From 37a4278bc5d9c1b4de2d43802268c02cd99900a5 Mon Sep 17 00:00:00 2001 From: ydshieh Date: Tue, 26 Aug 2025 11:17:59 +0200 Subject: [PATCH 3/6] up --- .github/workflows/push-important-models.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/push-important-models.yml b/.github/workflows/push-important-models.yml index 285f522d179c..2d317bfcb7e0 100644 --- a/.github/workflows/push-important-models.yml +++ b/.github/workflows/push-important-models.yml @@ -2,18 +2,7 @@ name: Slow tests on important models (on Push - A10) on: push: - branches: [ update_important_models ] - -env: - OUTPUT_SLACK_CHANNEL_ID: "C06L2SGMEEA" - HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }} - HF_HOME: /mnt/cache - TRANSFORMERS_IS_CI: yes - OMP_NUM_THREADS: 8 - MKL_NUM_THREADS: 8 - RUN_SLOW: yes # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access. # This token is created under the bot `hf-transformers-bot`. - SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }} - TF_FORCE_GPU_ALLOW_GROWTH: true + branches: [ main ] jobs: get_modified_models: From ee2e77284880844672c9e785b51d9837238bc9ef Mon Sep 17 00:00:00 2001 From: ydshieh Date: Tue, 26 Aug 2025 11:28:05 +0200 Subject: [PATCH 4/6] up --- utils/important_files.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 utils/important_files.py diff --git a/utils/important_files.py b/utils/important_files.py new file mode 100644 index 000000000000..f932d8d363f6 --- /dev/null +++ b/utils/important_files.py @@ -0,0 +1,28 @@ +# List here the models to always test. +IMPORTANT_MODELS = [ + "auto", + "bert", + "gpt2", + "t5", + "modernbert", + "vit,clip", + "detr", + "table_transformer", + "got_ocr2", + "whisper", + "wav2vec2", + "qwen2_audio", + "speech_t5", + "csm", + "llama", + "gemma3", + "qwen2", + "mistral3", + "qwen2_5_vl", + "llava", + "smolvlm", + "internvl", + "gemma3n", + "gpt_oss", + "qwen2_5_omni", +] From 64f04464b2939a3e6f7419946dbb523c117f49b2 Mon Sep 17 00:00:00 2001 From: ydshieh Date: Tue, 26 Aug 2025 12:06:08 +0200 Subject: [PATCH 5/6] up --- utils/tests_fetcher.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index bb188fee17f5..546a14fd9822 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -61,6 +61,9 @@ from git import Repo +# List here the models not to be filtered by `filter_tests`. +from important_files import IMPORTANT_MODELS + PATH_TO_REPO = Path(__file__).parent.parent.resolve() PATH_TO_EXAMPLES = PATH_TO_REPO / "examples" @@ -72,10 +75,6 @@ NUM_MODELS_TO_TRIGGER_FULL_CI = 30 -# List here the models not to be filtered by `filter_tests`. -from important_files import IMPORTANT_MODELS - - @contextmanager def checkout_commit(repo: Repo, commit_id: str): """ From 89cd6dc19123efbbc4a02821c0843b695134ff50 Mon Sep 17 00:00:00 2001 From: ydshieh Date: Wed, 27 Aug 2025 10:24:33 +0200 Subject: [PATCH 6/6] update --- .github/workflows/push-important-models.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push-important-models.yml b/.github/workflows/push-important-models.yml index 2d317bfcb7e0..f1b0f6a9554f 100644 --- a/.github/workflows/push-important-models.yml +++ b/.github/workflows/push-important-models.yml @@ -14,7 +14,7 @@ jobs: - name: Check out code uses: actions/checkout@v4 - - name: Get changed files using GitHub API + - name: Get changed files using `actions/github-script` id: get-changed-files uses: actions/github-script@v7 with: @@ -68,6 +68,8 @@ jobs: - name: Parse changed files with Python if: steps.get-changed-files.outputs.any_changed == 'true' + env: + CHANGED_FILES: ${{ steps.get-changed-files.outputs.changed_files }} id: set-matrix run: | python3 - << 'EOF' @@ -84,7 +86,7 @@ jobs: print(f"Important models: {IMPORTANT_MODELS}") # Get the changed files from the previous step - changed_files_str = "${{ steps.get-changed-files.outputs.changed_files }}" + changed_files_str = os.environ.get('CHANGED_FILES', '') changed_files = changed_files_str.split() if changed_files_str else [] # Filter to only Python files @@ -115,7 +117,7 @@ jobs: result_models.add(model_dir) print(f"Added model directory: {model_dir}") - # Check for specific files under src/transformers/ or generation files + # Check for specific files under src/transformers/ or src/transformers/generation/ files elif file.startswith('src/transformers/generation/') or \ (file.startswith('src/transformers/') and os.path.basename(file) in transformers_utils_files): print(f"Found core file: {file} - including all important models")