From 3272fc38b5050d8bea236590c48260dfba7a41b3 Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Wed, 9 Apr 2025 22:33:30 -0700 Subject: [PATCH 1/9] Run tests only in merge queue Signed-off-by: Charlie Truong --- .github/workflows/cicd-main.yml | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 4041f31bc3..0ceff3dbae 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -21,13 +21,6 @@ on: types: [labeled] merge_group: types: [checks_requested] - workflow_dispatch: - inputs: - test_to_run: - required: false - default: all - type: string - description: Comma-separated list of tests to run. Use "all" to run the full test suite. # TODO: Due to limited compute, disabling pushes to main. This is okay to do since we force PRs to be up to date and the CI tests on pull/$PR_NUM/merge #push: # branches: @@ -41,20 +34,8 @@ jobs: pre-flight: runs-on: ubuntu-latest outputs: - test_to_run: ${{ steps.test_to_run.outputs.main }} - all: ${{ steps.all.outputs.main }} run_ci: ${{ steps.evaluate.outputs.run_ci }} steps: - - name: Parse test_to_run - id: test_to_run - run: | - parsed_string=$(echo ${{ inputs.test_to_run || 'all' }} | jq -c --raw-input 'split(",")') - echo "main=${parsed_string}" | tee -a "$GITHUB_OUTPUT" - - name: Parse all - id: all - run: | - echo "main=${{ contains(fromJSON(steps.test_to_run.outputs.main), 'all') }}" | tee -a "$GITHUB_OUTPUT" - - name: Get changed files id: changed-files if: github.event_name == 'pull_request' @@ -75,19 +56,17 @@ jobs: CHANGED_DOCS: ${{ steps.changed-files.outputs.doc_all_changed_files }} CHANGED_SRC: ${{ steps.changed-files.outputs.src_all_changed_files }} IS_PULLREQUEST: ${{ github.event_name == 'pull_request' }} - LABEL: ${{ github.event.label.name == 'Run CICD' }} MERGE_GROUP: ${{ github.event_name == 'merge_group' }} run: | # Some output that's helpful for debugging echo "Docs changed: $CHANGED_DOCS" echo "Src changed: $CHANGED_SRC" - # echo "DOCS_ONLY: $DOCS_ONLY" - echo "LABEL: $LABEL" + echo "DOCS_ONLY: $DOCS_ONLY" echo "IS_PULLREQUEST: $IS_PULLREQUEST" # Run CI only (on main or if label is attached) and if it's not only docs - echo run_ci=$([[ ("$LABEL" = "true" || "$IS_PULLREQUEST" = "false" || "$MERGE_GROUP" = "true") && "$DOCS_ONLY" = "false" ]] && echo "true" || echo "false") | tee -a "$GITHUB_OUTPUT" + echo run_ci=$([[ ("$IS_PULLREQUEST" = "true" || "$MERGE_GROUP" = "true") && "$DOCS_ONLY" = "false" ]] && echo "true" || echo "false") | tee -a "$GITHUB_OUTPUT" lint-check: name: Lint check @@ -118,7 +97,7 @@ jobs: uv run --group docs sphinx-build . _build/html build-container: - if: ${{ needs.pre-flight.outputs.run_ci == 'true' }} + if: ${{ needs.pre-flight.outputs.run_ci == 'true' && github.event_name == 'merge_group' }} needs: [pre-flight] uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_build_container.yml@v0.22.7 with: @@ -134,7 +113,7 @@ jobs: name: Tests needs: [build-container, pre-flight] uses: ./.github/workflows/_run_test.yml - if: ${{ needs.pre-flight.outputs.run_ci == 'true' }} + if: ${{ needs.pre-flight.outputs.run_ci == 'true' && github.event_name == 'merge_group' }} with: RUNNER: self-hosted-azure TIMEOUT: 15 @@ -173,7 +152,7 @@ jobs: - name: main env: JOB_RESULTS: ${{ toJSON(needs) }} - ALL_SUCCESS: ${{ !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'skipped') }} + ALL_SUCCESS: ${{ (github.event_name == 'pull_request' && needs.lint-check.result == 'success' && needs.sphinx-build.result == 'success') || (github.event_name == 'merge_queue' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'skipped')) }} CI_SKIP: ${{ github.event.label.name == 'Skip CICD' }} run: | From 246bb22c8f342de28bce9a09c07e453802ab1e42 Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Wed, 9 Apr 2025 22:57:45 -0700 Subject: [PATCH 2/9] CI workflow runs without labels Signed-off-by: Charlie Truong --- .github/workflows/cicd-main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 0ceff3dbae..68f778cde4 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -18,7 +18,6 @@ on: branches: - "main" - "r**" - types: [labeled] merge_group: types: [checks_requested] # TODO: Due to limited compute, disabling pushes to main. This is okay to do since we force PRs to be up to date and the CI tests on pull/$PR_NUM/merge From 1edd32615e8c517375fe0940cc70cdf8cb7acb76 Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Wed, 9 Apr 2025 23:01:20 -0700 Subject: [PATCH 3/9] Minor change to trigger CI Signed-off-by: Charlie Truong --- .github/workflows/cicd-main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 68f778cde4..744e68fa09 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -61,8 +61,8 @@ jobs: echo "Docs changed: $CHANGED_DOCS" echo "Src changed: $CHANGED_SRC" - echo "DOCS_ONLY: $DOCS_ONLY" echo "IS_PULLREQUEST: $IS_PULLREQUEST" + echo "DOCS_ONLY: $DOCS_ONLY" # Run CI only (on main or if label is attached) and if it's not only docs echo run_ci=$([[ ("$IS_PULLREQUEST" = "true" || "$MERGE_GROUP" = "true") && "$DOCS_ONLY" = "false" ]] && echo "true" || echo "false") | tee -a "$GITHUB_OUTPUT" From 0117b2836944a72eaafd882ae68ea20b91ac1f1b Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Tue, 15 Apr 2025 20:07:53 -0500 Subject: [PATCH 4/9] Allow optionally running CI tests based on label Signed-off-by: Charlie Truong --- .github/workflows/cicd-main.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 69a6902338..d2eac91f99 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -67,6 +67,7 @@ jobs: CHANGED_DOCS: ${{ steps.changed-files.outputs.doc_all_changed_files }} CHANGED_SRC: ${{ steps.changed-files.outputs.src_all_changed_files }} IS_PULLREQUEST: ${{ github.event_name == 'pull_request' }} + LABEL: ${{ github.event.label.name }} MERGE_GROUP: ${{ github.event_name == 'merge_group' }} run: | # Some output that's helpful for debugging @@ -78,10 +79,10 @@ jobs: # Run CI only (on main or if label is attached) and if it's not only docs # Determine test level based on conditions - if [[ "$DOCS_ONLY" == "true" ]]; then + if [[ "$DOCS_ONLY" == "true" || "$LABEL" == "CI:docs" ]]; then # For doc-only changes, run only doc tests TEST_LEVEL="docs" - elif [[ "$LABEL" == "true" || "$IS_PULLREQUEST" == "false" || "$MERGE_GROUP" == "true" ]]; then + elif [[ "$LABEL" == "CI:L0" || "$IS_PULLREQUEST" == "false" || "$MERGE_GROUP" == "true" ]]; then # For labeled PRs, pushes to main (IS_PULL_REQUEST=false), or merge group events, run L0 by default TEST_LEVEL="L0" else @@ -101,7 +102,7 @@ jobs: name: Lint check needs: [pre-flight] runs-on: ubuntu-latest - if: ${{ needs.pre-flight.outputs.test_level != 'none' }} + if: ${{ contains(fromJson('["L0","L1","L2"]'), needs.pre-flight.outputs.test_level) }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -115,7 +116,7 @@ jobs: name: Sphinx build needs: [pre-flight] runs-on: ubuntu-latest - if: ${{ needs.pre-flight.outputs.test_level != 'none' }} + if: ${{ contains(fromJson('["docs","L0","L1","L2"]'), needs.pre-flight.outputs.test_level) }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -126,7 +127,7 @@ jobs: uv run --group docs sphinx-build . _build/html build-container: - if: ${{ needs.pre-flight.outputs.test_level != 'none' }} + if: ${{ contains(fromJson('["L0","L1","L2"]'), needs.pre-flight.outputs.test_level) }} needs: [pre-flight] uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_build_container.yml@v0.22.7 with: @@ -142,7 +143,7 @@ jobs: name: Tests needs: [build-container, pre-flight] uses: ./.github/workflows/_run_test.yml - if: ${{ needs.pre-flight.outputs.test_level != 'none' }} + if: ${{ contains(fromJson('["L0","L1","L2"]'), needs.pre-flight.outputs.test_level) }} with: RUNNER: self-hosted-azure TIMEOUT: 20 From a5a0d4c2578b9414420f7557ac7fccc5fdd5ecd2 Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Tue, 15 Apr 2025 20:16:19 -0500 Subject: [PATCH 5/9] Add additional CI labels Signed-off-by: Charlie Truong --- .github/workflows/cicd-main.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index d2eac91f99..9d33324469 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -83,8 +83,12 @@ jobs: # For doc-only changes, run only doc tests TEST_LEVEL="docs" elif [[ "$LABEL" == "CI:L0" || "$IS_PULLREQUEST" == "false" || "$MERGE_GROUP" == "true" ]]; then - # For labeled PRs, pushes to main (IS_PULL_REQUEST=false), or merge group events, run L0 by default + # For labeled PRs, pushes to main (IS_PULL_REQUEST=false), run L0 by default TEST_LEVEL="L0" + elif [[ "$LABEL" == "CI:L1" ]]; then + TEST_LEVEL="L1" + elif [[ "$LABEL" == "CI:L2" ]]; then + TEST_LEVEL="L2" else # Skip tests by default for non-labeled PRs TEST_LEVEL="none" @@ -127,7 +131,7 @@ jobs: uv run --group docs sphinx-build . _build/html build-container: - if: ${{ contains(fromJson('["L0","L1","L2"]'), needs.pre-flight.outputs.test_level) }} + if: ${{ contains(fromJson('["docs","L0","L1","L2"]'), needs.pre-flight.outputs.test_level) }} needs: [pre-flight] uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_build_container.yml@v0.22.7 with: From 0ebdc5955fd7b192bfe9f621508a850fdd693277 Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Tue, 15 Apr 2025 20:29:41 -0500 Subject: [PATCH 6/9] Always lint and build docs Signed-off-by: Charlie Truong --- .github/workflows/cicd-main.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 9d33324469..b03eecd1a4 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -18,6 +18,7 @@ on: branches: - "main" - "r**" + types: [labeled, opened, synchronize, reopened] merge_group: types: [checks_requested] workflow_dispatch: @@ -106,7 +107,6 @@ jobs: name: Lint check needs: [pre-flight] runs-on: ubuntu-latest - if: ${{ contains(fromJson('["L0","L1","L2"]'), needs.pre-flight.outputs.test_level) }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -120,7 +120,6 @@ jobs: name: Sphinx build needs: [pre-flight] runs-on: ubuntu-latest - if: ${{ contains(fromJson('["docs","L0","L1","L2"]'), needs.pre-flight.outputs.test_level) }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -214,11 +213,13 @@ jobs: # Job is considered successful if nothing was run, or if all jobs were successful (the tests run even if only docs were run b/c doctests are selected) ALL_SUCCESS: >- ${{ - (needs.pre-flight.outputs.test_level == 'none') || - (needs.pre-flight.outputs.test_level != 'none' && - needs.lint-check.result == 'success' && - needs.sphinx-build.result == 'success' && - needs.tests.result == 'success') + needs.lint-check.result == 'success' && + needs.sphinx-build.result == 'success' && + ( + needs.pre-flight.outputs.test_level == 'none' || + (needs.pre-flight.outputs.test_level != 'none' && + needs.tests.result == 'success') + ) }} CI_SKIP: ${{ github.event.label.name == 'Skip CICD' }} TEST_LEVEL: ${{ needs.pre-flight.outputs.test_level }} From 82547d45b0f024fa4d871603868505bed4f24410 Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Tue, 15 Apr 2025 20:34:51 -0500 Subject: [PATCH 7/9] Always build container and run tests if test level is not none Signed-off-by: Charlie Truong --- .github/workflows/cicd-main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index b03eecd1a4..0216e38f46 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -130,7 +130,7 @@ jobs: uv run --group docs sphinx-build . _build/html build-container: - if: ${{ contains(fromJson('["docs","L0","L1","L2"]'), needs.pre-flight.outputs.test_level) }} + if: ${{ needs.pre-flight.outputs.test_level != 'none' }} needs: [pre-flight] uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_build_container.yml@v0.22.7 with: @@ -146,7 +146,7 @@ jobs: name: Tests needs: [build-container, pre-flight] uses: ./.github/workflows/_run_test.yml - if: ${{ contains(fromJson('["L0","L1","L2"]'), needs.pre-flight.outputs.test_level) }} + if: ${{ needs.pre-flight.outputs.test_level != 'none' }} with: RUNNER: self-hosted-azure TIMEOUT: 20 From a4363a3249869d045458b8b7e9f0586d04168dbb Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Tue, 15 Apr 2025 20:36:40 -0500 Subject: [PATCH 8/9] Revert L0 test comment on merge group Signed-off-by: Charlie Truong --- .github/workflows/cicd-main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 0216e38f46..92fd361b19 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -84,7 +84,7 @@ jobs: # For doc-only changes, run only doc tests TEST_LEVEL="docs" elif [[ "$LABEL" == "CI:L0" || "$IS_PULLREQUEST" == "false" || "$MERGE_GROUP" == "true" ]]; then - # For labeled PRs, pushes to main (IS_PULL_REQUEST=false), run L0 by default + # For labeled PRs, pushes to main (IS_PULL_REQUEST=false), or merge group events, run L0 by default TEST_LEVEL="L0" elif [[ "$LABEL" == "CI:L1" ]]; then TEST_LEVEL="L1" From 80c5b543dcd66feb5c0c78dde0900a0ef65ac15c Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Tue, 15 Apr 2025 20:47:33 -0500 Subject: [PATCH 9/9] Only build docs if test level is not none Signed-off-by: Charlie Truong --- .github/workflows/cicd-main.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 92fd361b19..8fc9308931 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -117,6 +117,7 @@ jobs: pre-commit run --all-files --show-diff-on-failure --color=always sphinx-build: + if: ${{ needs.pre-flight.outputs.test_level != 'none' }} name: Sphinx build needs: [pre-flight] runs-on: ubuntu-latest @@ -214,11 +215,13 @@ jobs: ALL_SUCCESS: >- ${{ needs.lint-check.result == 'success' && - needs.sphinx-build.result == 'success' && ( needs.pre-flight.outputs.test_level == 'none' || - (needs.pre-flight.outputs.test_level != 'none' && - needs.tests.result == 'success') + ( + needs.pre-flight.outputs.test_level != 'none' && + needs.sphinx-build.result == 'success' && + needs.tests.result == 'success' + ) ) }} CI_SKIP: ${{ github.event.label.name == 'Skip CICD' }}