From 8f468d7bb2170d622199a743ef3fe8846a95f9d4 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 24 Apr 2023 13:34:45 +0200 Subject: [PATCH] Speed up verification and instalation of providers in CI For PRs that are just changing selected providers, there is no need to install and verify all providers, just those affected should be installed (and full verification should only be done when all providers are being built - so generally speaking, when Core PRs are run or when canary builds are run. This PR distinguishes those two cases: * installing and verification when all providers are selected * installing only affected providers when only subset of them are affected Also a bug was found where core file changes did not trigger provider docs build during implementation. This has been fixed --- .github/workflows/ci.yml | 153 +++++++---- dev/breeze/SELECTIVE_CHECKS.md | 102 ++++---- .../airflow_breeze/utils/selective_checks.py | 50 +++- dev/breeze/tests/test_selective_checks.py | 242 +++++++++++++----- 4 files changed, 380 insertions(+), 167 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e4d476ff29c1..084b3e3ed4588 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,6 +128,8 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} outputs: cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} + affected-providers-list-as-string: > + ${{ steps.selective-checks.outputs.affected-providers-list-as-string }} upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} python-versions: ${{ steps.selective-checks.outputs.python-versions }} python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} @@ -136,7 +138,8 @@ jobs: default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} kubernetes-versions-list-as-string: >- ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} - kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} + kubernetes-combos-list-as-string: >- + ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} @@ -146,7 +149,8 @@ jobs: default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} - parallel-test-types: ${{ steps.selective-checks.outputs.parallel-test-types }} + parallel-test-types-list-as-string: >- + ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }} postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }} @@ -164,7 +168,7 @@ jobs: needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} default-branch: ${{ steps.selective-checks.outputs.default-branch }} default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} - docs-filter: ${{ steps.selective-checks.outputs.docs-filter }} + docs-filter-list-as-string: ${{ steps.selective-checks.outputs.docs-filter-list-as-string }} skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} @@ -551,7 +555,7 @@ jobs: env: PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" BACKEND: sqlite - PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" - name: "Fix ownership" run: breeze ci fix-ownership if: always() @@ -686,7 +690,7 @@ jobs: docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }} docs-inventory- - name: "Build docs" - run: breeze build-docs ${{ needs.build-info.outputs.docs-filter }} + run: breeze build-docs ${{ needs.build-info.outputs.docs-filter-list-as-string }} - name: Configure AWS credentials uses: ./.github/actions/configure-aws-credentials if: > @@ -728,11 +732,13 @@ jobs: - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare provider documentation" - run: breeze release-management prepare-provider-documentation + run: > + breeze release-management prepare-provider-documentation + ${{ needs.build-info.outputs.affected-providers-list-as-string }} - name: "Prepare provider packages: wheel" run: > breeze release-management prepare-provider-packages --version-suffix-for-pypi dev0 - --package-format wheel + --package-format wheel ${{ needs.build-info.outputs.affected-providers-list-as-string }} - name: "Prepare airflow package: wheel" run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0 - name: "Verify wheel packages with twine" @@ -741,10 +747,18 @@ jobs: run: > breeze release-management generate-issue-content-providers --only-available-in-dist --disable-progress - - name: "Install and test provider packages and airflow via wheel files" + - name: "Install and verify all provider packages and airflow via wheel files" run: > breeze release-management verify-provider-packages --use-packages-from-dist --package-format wheel --use-airflow-version wheel + if: needs.build-info.outputs.affected-providers-list-as-string == '' + env: + SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" + - name: "Install affected provider packages and airflow via wheel files" + run: > + breeze release-management install-provider-packages + --package-format wheel --use-airflow-version wheel --run-in-parallel + if: needs.build-info.outputs.affected-providers-list-as-string != '' env: SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" - name: "Fix ownership" @@ -776,7 +790,7 @@ jobs: - name: "Prepare provider packages: wheel" run: > breeze release-management prepare-provider-packages --version-suffix-for-pypi dev0 - --package-format wheel + --package-format wheel ${{ needs.build-info.outputs.affected-providers-list-as-string }} - name: "Fix incompatible 2.3 provider packages" run: | # This step should remove the provider packages that are not compatible with 2.3 @@ -801,10 +815,16 @@ jobs: providers = json.loads(Path("generated/provider_dependencies.json").read_text()) provider_keys = ",".join(providers.keys()) print("AIRFLOW_EXTRAS={}".format(provider_keys))' >> $GITHUB_ENV - - name: "Install and verify provider packages and airflow on Airflow 2.3 files" + - name: "Install and verify all provider packages and airflow on Airflow 2.3 files" run: > breeze release-management verify-provider-packages --use-airflow-version 2.3.0 --use-packages-from-dist --airflow-constraints-reference constraints-2.3.0 + if: needs.build-info.outputs.affected-providers-list-as-string == '' + - name: "Install affected provider packages and airflow on Airflow 2.3 files" + run: > + breeze release-management install-provider-packages --use-airflow-version 2.3.0 + --airflow-constraints-reference constraints-2.3.0 --run-in-parallel + if: needs.build-info.outputs.affected-providers-list-as-string != '' - name: "Fix ownership" run: breeze ci fix-ownership if: always() @@ -834,19 +854,24 @@ jobs: - name: "Prepare provider packages: sdist" run: > breeze release-management prepare-provider-packages - --version-suffix-for-pypi dev0 - --package-format sdist + --version-suffix-for-pypi dev0 --package-format sdist + ${{ needs.build-info.outputs.affected-providers-list-as-string }} - name: "Prepare airflow package: sdist" run: > breeze release-management prepare-airflow-package - --version-suffix-for-pypi dev0 - --package-format sdist + --version-suffix-for-pypi dev0 --package-format sdist - name: "Verify sdist packages with twine" run: pipx install twine --force && twine check dist/*.tar.gz - - name: "Install provider packages and airflow via sdist files" + - name: "Install all provider packages and airflow via sdist files" + run: > + breeze release-management install-provider-packages + --package-format sdist --use-airflow-version sdist --run-in-parallel + if: needs.build-info.outputs.affected-providers-list-as-string == '' + - name: "Install affected provider packages and airflow via sdist files" run: > breeze release-management install-provider-packages --package-format sdist --use-airflow-version sdist --run-in-parallel + if: needs.build-info.outputs.affected-providers-list-as-string != '' - name: "Fix ownership" run: breeze ci fix-ownership if: always() @@ -927,7 +952,7 @@ jobs: timeout-minutes: 130 name: > Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: - ${{needs.build-info.outputs.parallel-test-types}} + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} runs-on: "${{needs.build-info.outputs.runs-on}}" needs: [build-info, wait-for-ci-images] strategy: @@ -938,7 +963,7 @@ jobs: fail-fast: false env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" @@ -959,14 +984,19 @@ jobs: persist-credentials: false - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" uses: ./.github/actions/prepare_breeze_and_image - - name: "Migration Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Migration Tests: + ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/migration_tests - - name: "Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} run: breeze testing tests --run-in-parallel - name: "Tests ARM Pytest collection: ${{matrix.python-version}}" run: breeze testing tests --run-in-parallel --collect-only --remove-arm-packages if: matrix.postgres-version == needs.build-info.outputs.default-postgres-version - - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Post Tests: + ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/post_tests tests-postgres-boto: @@ -974,12 +1004,12 @@ jobs: name: > LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}}, Py${{needs.build-info.outputs.default-python-version}}: - ${{needs.build-info.outputs.parallel-test-types}} + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} runs-on: "${{needs.build-info.outputs.runs-on}}" needs: [build-info, wait-for-ci-images] env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" @@ -1007,11 +1037,11 @@ jobs: uses: ./.github/actions/prepare_breeze_and_image - name: > Tests: ${{needs.build-info.outputs.default-python-version}}: - ${{needs.build-info.outputs.parallel-test-types}} + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} run: breeze testing tests --run-in-parallel - name: > Post Tests: ${{needs.build-info.outputs.default-python-version}}: - ${{needs.build-info.outputs.parallel-test-types}} + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/post_tests tests-postgres-in-progress-features-disabled: @@ -1019,12 +1049,12 @@ jobs: name: > InProgressDisabledPostgres${{needs.build-info.outputs.default-postgres-version}}, Py${{needs.build-info.outputs.default-python-version}}: - ${{needs.build-info.outputs.parallel-test-types}} + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} runs-on: "${{needs.build-info.outputs.runs-on}}" needs: [build-info, wait-for-ci-images] env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" @@ -1053,18 +1083,18 @@ jobs: uses: ./.github/actions/prepare_breeze_and_image - name: > Tests: ${{needs.build-info.outputs.default-python-version}}: - ${{needs.build-info.outputs.parallel-test-types}} + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} run: breeze testing tests --run-in-parallel - name: > Post Tests: ${{needs.build-info.outputs.default-python-version}}: - ${{needs.build-info.outputs.parallel-test-types}} + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/post_tests tests-mysql: timeout-minutes: 130 name: > MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: - ${{needs.build-info.outputs.parallel-test-types}} + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} runs-on: "${{needs.build-info.outputs.runs-on}}" needs: [build-info, wait-for-ci-images] strategy: @@ -1077,7 +1107,7 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" - PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" BACKEND: "mysql" PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" @@ -1095,18 +1125,23 @@ jobs: persist-credentials: false - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" uses: ./.github/actions/prepare_breeze_and_image - - name: "Migration Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Migration Tests: + ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/migration_tests - - name: "Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} run: breeze testing tests --run-in-parallel - - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Post Tests: + ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/post_tests tests-mssql: timeout-minutes: 130 name: > MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: - ${{needs.build-info.outputs.parallel-test-types}} + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} runs-on: "${{needs.build-info.outputs.runs-on}}" needs: [build-info, wait-for-ci-images] strategy: @@ -1117,7 +1152,7 @@ jobs: fail-fast: false env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" @@ -1138,17 +1173,22 @@ jobs: persist-credentials: false - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" uses: ./.github/actions/prepare_breeze_and_image - - name: "Migration Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Migration Tests: + ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/migration_tests - - name: "Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} run: breeze testing tests --run-in-parallel - - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Post Tests: + ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/post_tests tests-sqlite: timeout-minutes: 130 name: > - Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.parallel-test-types}} + Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.parallel-test-types-list-as-string}} runs-on: "${{needs.build-info.outputs.runs-on}}" needs: [build-info, wait-for-ci-images] strategy: @@ -1159,7 +1199,7 @@ jobs: if: needs.build-info.outputs.run-tests == 'true' env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" @@ -1178,11 +1218,16 @@ jobs: persist-credentials: false - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" uses: ./.github/actions/prepare_breeze_and_image - - name: "Migration Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Migration Tests: + ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/migration_tests - - name: "Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} run: breeze testing tests --run-in-parallel - - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Post Tests: + ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/post_tests tests-integration-postgres: @@ -1192,7 +1237,7 @@ jobs: needs: [build-info, wait-for-ci-images] env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" @@ -1247,7 +1292,9 @@ jobs: - name: "Integration Tests Postgres: all-testable" run: breeze testing integration-tests --integration all-testable if: needs.build-info.outputs.runs-on == 'self-hosted' - - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Post Tests: + ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/post_tests tests-integration-mysql: @@ -1257,7 +1304,7 @@ jobs: needs: [build-info, wait-for-ci-images] env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" @@ -1285,7 +1332,9 @@ jobs: - name: "Integration Tests MySQL: all-testable" run: breeze testing integration-tests --integration all-testable if: needs.build-info.outputs.runs-on == 'self-hosted' - - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}" + - name: > + Post Tests: + ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} uses: ./.github/actions/post_tests if: needs.build-info.outputs.runs-on == 'self-hosted' @@ -1410,7 +1459,8 @@ jobs: fetch-depth: 2 persist-credentials: false - name: > - Prepare breeze & PROD image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} + Prepare breeze & PROD image: + ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image with: pull-image-type: 'PROD' @@ -1456,21 +1506,22 @@ jobs: key: "\ k8s-env-${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','setup.cfg',\ 'setup.py','pyproject.toml','generated/provider_dependencies.json') }}" - - name: Run complete K8S tests ${{needs.build-info.outputs.kubernetes-combos}} + - name: Run complete K8S tests ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} run: breeze k8s run-complete-tests --run-in-parallel --upgrade env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} EXECUTOR: ${{matrix.executor}} VERBOSE: false - - name: Upload KinD logs on failure ${{needs.build-info.outputs.kubernetes-combos}} + - name: Upload KinD logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} uses: actions/upload-artifact@v3 if: failure() || cancelled() with: name: kind-logs-${{matrix.executor}} path: /tmp/kind_logs_* retention-days: 7 - - name: Upload test resource logs on failure ${{needs.build-info.outputs.kubernetes-combos}} + - name: > + Upload test resource logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} uses: actions/upload-artifact@v3 if: failure() || cancelled() with: diff --git a/dev/breeze/SELECTIVE_CHECKS.md b/dev/breeze/SELECTIVE_CHECKS.md index 0c6bfa51b3511..a3e64a69774c7 100644 --- a/dev/breeze/SELECTIVE_CHECKS.md +++ b/dev/breeze/SELECTIVE_CHECKS.md @@ -103,57 +103,61 @@ The logic implements the following rules: * if `Image building` is disabled, only basic pre-commits are enabled - no 'image-depending` pre-commits are enabled. * If there are some setup files changed, `upgrade to newer dependencies` is enabled. -* If docs are build, the `docs-filter` will determine which docs packages to build. This is based on +* If docs are build, the `docs-filter-list-as-string` will determine which docs packages to build. This is based on several criteria: if any of the airflow core, charts, docker-stack, providers files or docs have changed, then corresponding packages are build (including cross-dependent providers). If any of the core files changed, also providers docs are built because all providers depend on airflow docs. If any of the docs build python files changed or when build is "canary" type in main - all docs packages are built. -The selective check outputs available are described below: - -| Output | Meaning of the output | Example value | -|------------------------------------|---------------------------------------------------------------------------------------------------------|-----------------------------------------------------| -| all-python-versions | List of all python versions there are available in the form of JSON array | ['3.7', '3.8', '3.9', '3.10'] | -| all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.7 3.8 3.9 3.10 | -| basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | -| cache-directive | Which cache should be be used for images ("registry", "local" , "disabled") | registry | -| debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | -| default-branch | Which branch is default for the the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | -| default-constraints-branch | Which branch is default for the the build ("constraints-main" for main branch, "constraints-2-4" etc.) | constraints-main | -| default-helm-version | Which Helm version to use as default | v3.9.4 | -| default-kind-version | Which Kind version to use as default | v0.16.0 | -| default-kubernetes-version | Which Kubernetes version to use as default | v1.25.2 | -| default-mssql-version | Which MsSQL version to use as default | 2017-latest | -| default-mysql-version | Which MySQL version to use as default | 5.7 | -| default-postgres-version | Which Postgres version to use as default | 10 | -| default-python-version | Which Python version to use as default | 3.7 | -| docs-build | Whether to build documentation ("true"/"false") | true | -| docs-filter | What filter to apply to docs building - based on which documentation packages should be built | --package-filter apache-airflow - | -| full-tests-needed | Whether this build runs complete set of tests or only subset (for faster PR builds) | false | -| helm-version | Which Helm version to use for tests | v3.9.4 | -| image-build | Whether CI image build is needed | true | -| kind-version | Which Kind version to use for tests | v0.16.0 | -| kubernetes-combos | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.7-v1.25.2 3.8-v1.26.4 | -| kubernetes-versions | All Kubernetes versions to use for tests as JSON array | ['v1.25.2'] | -| kubernetes-versions-list-as-string | All Kubernetes versions to use for tests as space-separated string | v1.25.2 | -| mssql-exclude | Which versions of MsSQL to exclude for tests as JSON array | [] | -| mssql-versions | Which versions of MsSQL to use for tests as JSON array | ['2017-latest'] | -| mysql-exclude | Which versions of MySQL to exclude for tests as JSON array | [] | -| mysql-versions | Which versions of MySQL to use for tests as JSON array | ['5.7'] | -| needs-api-codegen | Whether "api-codegen" are needed to run ("true"/"false") | true | -| needs-api-tests | Whether "api-tests" are needed to run ("true"/"false") | true | -| needs-helm-tests | Whether Helm tests are needed to run ("true"/"false") | true | -| needs-javascript-scans | Whether javascript CodeQL scans should be run ("true"/"false") | true | -| needs-python-scans | Whether Python CodeQL scans should be run ("true"/"false") | true | -| parallel-test-types | Which test types should be run for unit tests | API Always Providers\[amazon\] Providers\[-amazon\] | -| postgres-exclude | Which versions of Postgres to exclude for tests as JSON array | [] | -| postgres-versions | Which versions of Postgres to use for tests as JSON array | ['10'] | -| python-versions | Which versions of Python to use for tests as JSON array | ['3.7'] | -| python-versions-list-as-string | Which versions of MySQL to use for tests as space-separated string | 3.7 | -| run-kubernetes-tests | Whether Kubernetes tests should be run ("true"/"false") | true | -| run-tests | Whether unit tests should be run ("true"/"false") | true | -| run-www-tests | Whether WWW tests should be run ("true"/"false") | true | -| skip-pre-commits | Which pre-commits should be skipped during the static-checks run | identity | -| skip-provider-tests | When provider tests should be skipped (on non-main branch) | identity | -| sqlite-exclude | Which versions of Sqlite to exclude for tests as JSON array | [] | -| upgrade-to-newer-dependencies | Whether the image build should attempt to upgrade all dependencies (might be true/false or commit hash) | false | +The selective check outputs available are described below. In case of `list-as-string` values, +empty string means `everything`, where lack of the output means `nothing` and list elements are +separated by spaces. This is to accommodate for the wau how outputs of this kind can be easily used by +Github Actions to pass the list of parameters to a command to execute + +| Output | Meaning of the output | Example value | List as string | +|------------------------------------|---------------------------------------------------------------------------------------------------------|------------------------------------------------------------|----------------| +| affected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * | +| all-python-versions | List of all python versions there are available in the form of JSON array | ['3.7', '3.8', '3.9', '3.10'] | | +| all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.7 3.8 3.9 3.10 | * | +| basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | +| cache-directive | Which cache should be be used for images ("registry", "local" , "disabled") | registry | | +| debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | +| default-branch | Which branch is default for the the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | +| default-constraints-branch | Which branch is default for the the build ("constraints-main" for main branch, "constraints-2-4" etc.) | constraints-main | | +| default-helm-version | Which Helm version to use as default | v3.9.4 | | +| default-kind-version | Which Kind version to use as default | v0.16.0 | | +| default-kubernetes-version | Which Kubernetes version to use as default | v1.25.2 | | +| default-mssql-version | Which MsSQL version to use as default | 2017-latest | | +| default-mysql-version | Which MySQL version to use as default | 5.7 | | +| default-postgres-version | Which Postgres version to use as default | 10 | | +| default-python-version | Which Python version to use as default | 3.7 | | +| docs-build | Whether to build documentation ("true"/"false") | true | | +| docs-filter-list-as-string | What filter to apply to docs building - based on which documentation packages should be built | --package-filter apache-airflow -package-filter-helm-chart | | +| full-tests-needed | Whether this build runs complete set of tests or only subset (for faster PR builds) | false | | +| helm-version | Which Helm version to use for tests | v3.9.4 | | +| image-build | Whether CI image build is needed | true | | +| kind-version | Which Kind version to use for tests | v0.16.0 | | +| kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.7-v1.25.2 3.8-v1.26.4 | * | +| kubernetes-versions | All Kubernetes versions to use for tests as JSON array | ['v1.25.2'] | | +| kubernetes-versions-list-as-string | All Kubernetes versions to use for tests as space-separated string | v1.25.2 | * | +| mssql-exclude | Which versions of MsSQL to exclude for tests as JSON array | [] | | +| mssql-versions | Which versions of MsSQL to use for tests as JSON array | ['2017-latest'] | | +| mysql-exclude | Which versions of MySQL to exclude for tests as JSON array | [] | | +| mysql-versions | Which versions of MySQL to use for tests as JSON array | ['5.7'] | | +| needs-api-codegen | Whether "api-codegen" are needed to run ("true"/"false") | true | | +| needs-api-tests | Whether "api-tests" are needed to run ("true"/"false") | true | | +| needs-helm-tests | Whether Helm tests are needed to run ("true"/"false") | true | | +| needs-javascript-scans | Whether javascript CodeQL scans should be run ("true"/"false") | true | | +| needs-python-scans | Whether Python CodeQL scans should be run ("true"/"false") | true | | +| parallel-test-types-list-as-string | Which test types should be run for unit tests | API Always Providers\[amazon\] Providers\[-amazon\] | * | +| postgres-exclude | Which versions of Postgres to exclude for tests as JSON array | [] | | +| postgres-versions | Which versions of Postgres to use for tests as JSON array | ['10'] | | +| python-versions | Which versions of Python to use for tests as JSON array | ['3.7'] | | +| python-versions-list-as-string | Which versions of MySQL to use for tests as space-separated string | 3.7 | * | +| run-kubernetes-tests | Whether Kubernetes tests should be run ("true"/"false") | true | | +| run-tests | Whether unit tests should be run ("true"/"false") | true | | +| run-www-tests | Whether WWW tests should be run ("true"/"false") | true | | +| skip-pre-commits | Which pre-commits should be skipped during the static-checks run | true | | +| skip-provider-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | +| sqlite-exclude | Which versions of Sqlite to exclude for tests as JSON array | [] | | +| upgrade-to-newer-dependencies | Whether the image build should attempt to upgrade all dependencies (might be true/false or commit hash) | false | | diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index a54e7c86e1806..9a6c1dee44507 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -318,7 +318,9 @@ def __str__(self) -> str: output = [] for field_name in dir(self): if not field_name.startswith("_"): - output.append(get_ga_output(field_name, getattr(self, field_name))) + value = getattr(self, field_name) + if value is not None: + output.append(get_ga_output(field_name, value)) return "\n".join(output) default_python_version = DEFAULT_PYTHON_MAJOR_MINOR_VERSION @@ -453,7 +455,7 @@ def kubernetes_versions_list_as_string(self) -> str: return " ".join(self.kubernetes_versions) @cached_property - def kubernetes_combos(self) -> str: + def kubernetes_combos_list_as_string(self) -> str: python_version_array: list[str] = self.python_versions_list_as_string.split(" ") kubernetes_version_array: list[str] = self.kubernetes_versions_list_as_string.split(" ") combo_titles, short_combo_titles, combos = get_kubernetes_python_combos( @@ -519,7 +521,12 @@ def run_www_tests(self) -> bool: @cached_property def run_amazon_tests(self) -> bool: - return "amazon" in self.parallel_test_types or "Providers" in self.parallel_test_types.split(" ") + if self.parallel_test_types_list_as_string is None: + return False + return ( + "amazon" in self.parallel_test_types_list_as_string + or "Providers" in self.parallel_test_types_list_as_string.split(" ") + ) @cached_property def run_kubernetes_tests(self) -> bool: @@ -551,6 +558,11 @@ def _select_test_type_if_matching( get_console().print(f"[warning]{test_type} added because it matched {count} files[/]") return matched_files + def _are_all_providers_affected(self) -> bool: + # if "Providers" test is present in the list of tests, it means that we should run all providers tests + # prepare all providers packages and build all providers documentation + return "Providers" in self._get_test_types_to_run() + def _get_test_types_to_run(self) -> list[str]: candidate_test_types: set[str] = {"Always"} matched_files: set[str] = set() @@ -629,9 +641,9 @@ def _extract_long_provider_tests(current_test_types: set[str]): current_test_types.add(f"Providers[{','.join(provider_tests_to_run)}]") @cached_property - def parallel_test_types(self) -> str: + def parallel_test_types_list_as_string(self) -> str | None: if not self.run_tests: - return "" + return None if self.full_tests_needed: current_test_types = set(all_selective_test_types()) else: @@ -677,18 +689,22 @@ def upgrade_to_newer_dependencies(self) -> bool: ) > 0 or self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE] @cached_property - def docs_filter(self) -> str: + def docs_filter_list_as_string(self) -> str | None: + _ALL_DOCS_LIST = "" + if not self.docs_build: + return None if self._default_branch != "main": return "--package-filter apache-airflow --package-filter docker-stack" if self.full_tests_needed: - return "" + return _ALL_DOCS_LIST providers_affected = find_all_providers_affected(changed_files=self._files, include_docs=True) if ( providers_affected == "ALL_PROVIDERS" or "docs/conf.py" in self._files or "docs/build_docs.py" in self._files + or self._are_all_providers_affected() ): - return "" + return _ALL_DOCS_LIST packages = [] if any([file.startswith("airflow/") for file in self._files]): packages.append("apache-airflow") @@ -707,7 +723,9 @@ def skip_pre_commits(self) -> str: @cached_property def skip_provider_tests(self) -> bool: - return self._default_branch != "main" + return self._default_branch != "main" or not any( + test_type.startswith("Providers") for test_type in self._get_test_types_to_run() + ) @cached_property def cache_directive(self) -> str: @@ -720,3 +738,17 @@ def debug_resources(self) -> bool: @cached_property def helm_test_packages(self) -> str: return json.dumps(all_helm_test_packages()) + + @cached_property + def affected_providers_list_as_string(self) -> str | None: + _ALL_PROVIDERS_LIST = "" + if self.full_tests_needed: + return _ALL_PROVIDERS_LIST + if self._are_all_providers_affected(): + return _ALL_PROVIDERS_LIST + affected_providers = find_all_providers_affected(changed_files=self._files, include_docs=True) + if not affected_providers: + return None + if affected_providers == "ALL_PROVIDERS": + return _ALL_PROVIDERS_LIST + return " ".join(sorted(affected_providers)) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index aa475df886dcb..6597150d5e29b 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -26,17 +26,25 @@ ANSI_COLORS_MATCHER = re.compile(r"(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]") +ALL_DOCS_SELECTED_FOR_BUILD = "" +ALL_PROVIDERS_AFFECTED = "" + + def escape_ansi_colors(line): return ANSI_COLORS_MATCHER.sub("", line) def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): + escaped_stderr = escape_ansi_colors(stderr) for name, value in expected_outputs.items(): - search_string = rf"^{re.escape(name)}={re.escape(value)}$" - escaped_stderr = escape_ansi_colors(stderr) - assert re.search( - search_string, escaped_stderr, re.MULTILINE - ), f"Expected {name}={value} not found in {escaped_stderr}" + if value is None: + search_string = rf"^{re.escape(name)}=" + if re.search(search_string, escaped_stderr, re.MULTILINE): + raise AssertionError(f"The {name} output should not be in {escaped_stderr}") + else: + search_string = rf"^{re.escape(name)}={re.escape(value)}$" + if not re.search(search_string, escaped_stderr, re.MULTILINE): + raise AssertionError(f"Expected {name}={value} not found in {escaped_stderr}") @pytest.mark.parametrize( @@ -46,6 +54,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("INTHEWILD.md",), { + "affected-providers-list-as-string": None, "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -56,7 +65,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "false", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "", + "parallel-test-types-list-as-string": None, }, id="No tests on simple change", ) @@ -65,6 +74,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("airflow/api/file.py",), { + "affected-providers-list-as-string": None, "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -75,7 +85,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "API Always", + "parallel-test-types-list-as-string": "API Always", }, id="Only API tests and DOCS should run", ) @@ -87,6 +97,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "tests/providers/postgres/file.py", ), { + "affected-providers-list-as-string": "amazon common.sql google postgres", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -97,7 +108,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Providers[amazon] " + "parallel-test-types-list-as-string": "Providers[amazon] " "API Always Providers[common.sql,postgres] Providers[google]", }, id="API and providers tests and docs should run", @@ -107,6 +118,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("tests/providers/apache/beam/file.py",), { + "affected-providers-list-as-string": "apache.beam google", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -118,7 +130,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "false", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Always Providers[apache.beam] Providers[google]", + "parallel-test-types-list-as-string": "Always Providers[apache.beam] Providers[google]", }, id="Selected Providers and docs should run", ) @@ -127,6 +139,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("docs/file.rst",), { + "affected-providers-list-as-string": None, "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -138,7 +151,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "true", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "", + "parallel-test-types-list-as-string": None, }, id="Only docs builds should run - no tests needed", ) @@ -150,6 +163,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "tests/providers/postgres/file.py", ), { + "affected-providers-list-as-string": "amazon common.sql google postgres", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -161,7 +175,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Providers[amazon] " + "parallel-test-types-list-as-string": "Providers[amazon] " "Always Providers[common.sql,postgres] Providers[google]", }, id="Helm tests, providers (both upstream and downstream)," @@ -176,6 +190,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "tests/providers/http/file.py", ), { + "affected-providers-list-as-string": "airbyte apache.livy " + "dbt.cloud dingding discord http", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -187,7 +203,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Always " + "parallel-test-types-list-as-string": "Always " "Providers[airbyte,apache.livy,dbt.cloud,dingding,discord,http]", }, id="Helm tests, http and all relevant providers, kubernetes tests and " @@ -202,6 +218,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "tests/providers/airbyte/file.py", ), { + "affected-providers-list-as-string": "airbyte http", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -213,7 +230,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Always Providers[airbyte,http]", + "parallel-test-types-list-as-string": "Always Providers[airbyte,http]", }, id="Helm tests, airbyte/http providers, kubernetes tests and " "docs should run even if unimportant files were added", @@ -227,6 +244,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "tests/system/utils/file.py", ), { + "affected-providers-list-as-string": None, "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -238,7 +256,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Always", + "parallel-test-types-list-as-string": "Always", }, id="Docs should run even if unimportant files were added", ) @@ -247,6 +265,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("setup.py",), { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", "python-versions": "['3.7', '3.8', '3.9', '3.10']", @@ -257,7 +276,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "true", - "parallel-test-types": "Core Providers[-amazon,google] Other Providers[amazon] WWW " + "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " + "Other Providers[amazon] WWW " "API Always CLI Providers[google]", }, id="Everything should run - including all providers and upgrading to " @@ -268,6 +288,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("generated/provider_dependencies.json",), { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", "python-versions": "['3.7', '3.8', '3.9', '3.10']", @@ -278,7 +299,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "true", - "parallel-test-types": "Core Providers[-amazon,google] Other Providers[amazon] WWW " + "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " + "Other Providers[amazon] WWW " "API Always CLI Providers[google]", }, id="Everything should run and upgrading to newer requirements as dependencies change", @@ -287,6 +309,9 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("airflow/providers/amazon/__init__.py",), { + "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " + "common.sql exasol ftp google imap " + "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -298,7 +323,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "run-amazon-tests": "true", - "parallel-test-types": "Providers[amazon] Always " + "parallel-test-types-list-as-string": "Providers[amazon] Always " "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,imap," "mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, @@ -307,6 +332,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("tests/providers/airbyte/__init__.py",), { + "affected-providers-list-as-string": "airbyte http", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -318,13 +344,16 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "false", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Always Providers[airbyte,http]", + "parallel-test-types-list-as-string": "Always Providers[airbyte,http]", }, id="Providers tests tests run without amazon tests if no amazon file changed", ), pytest.param( ("airflow/providers/amazon/file.py",), { + "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " + "common.sql exasol ftp google imap " + "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "python-versions": "['3.7']", @@ -336,7 +365,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "true", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Providers[amazon] Always " + "parallel-test-types-list-as-string": "Providers[amazon] Always " "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp," "imap,mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, @@ -367,6 +396,7 @@ def test_expected_output_pull_request_main( ("full tests needed",), "main", { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", "python-versions": "['3.7', '3.8', '3.9', '3.10']", @@ -374,10 +404,11 @@ def test_expected_output_pull_request_main( "image-build": "true", "run-tests": "true", "docs-build": "true", - "docs-filter": "", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Core Providers[-amazon,google] Other Providers[amazon] WWW " + "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " + "Other Providers[amazon] WWW " "API Always CLI Providers[google]", }, id="Everything should run including all providers when full tests are needed", @@ -392,6 +423,7 @@ def test_expected_output_pull_request_main( ), "main", { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", "python-versions": "['3.7', '3.8', '3.9', '3.10']", @@ -399,10 +431,11 @@ def test_expected_output_pull_request_main( "image-build": "true", "run-tests": "true", "docs-build": "true", - "docs-filter": "", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Core Providers[-amazon,google] Other Providers[amazon] WWW " + "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " + "Other Providers[amazon] WWW " "API Always CLI Providers[google]", }, id="Everything should run including full providers when full " @@ -415,6 +448,7 @@ def test_expected_output_pull_request_main( ("full tests needed",), "main", { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", "python-versions": "['3.7', '3.8', '3.9', '3.10']", @@ -422,10 +456,11 @@ def test_expected_output_pull_request_main( "image-build": "true", "run-tests": "true", "docs-build": "true", - "docs-filter": "", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Core Providers[-amazon,google] Other Providers[amazon] WWW " + "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " + "Other Providers[amazon] WWW " "API Always CLI Providers[google]", }, id="Everything should run including full providers when" @@ -438,6 +473,7 @@ def test_expected_output_pull_request_main( ("full tests needed",), "v2-3-stable", { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", "python-versions": "['3.7', '3.8', '3.9', '3.10']", @@ -445,9 +481,11 @@ def test_expected_output_pull_request_main( "image-build": "true", "run-tests": "true", "docs-build": "true", + "docs-filter-list-as-string": "--package-filter apache-airflow " + "--package-filter docker-stack", "full-tests-needed": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types": "Core Other WWW API Always CLI", + "parallel-test-types-list-as-string": "Core Other WWW API Always CLI", }, id="Everything should run except Providers when full tests are needed for non-main branch", ) @@ -476,16 +514,18 @@ def test_expected_output_full_tests_needed( pytest.param( ("INTHEWILD.md",), { + "affected-providers-list-as-string": None, "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "image-build": "false", "needs-helm-tests": "false", "run-tests": "false", "docs-build": "false", + "docs-filter-list-as-string": None, "full-tests-needed": "false", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", - "parallel-test-types": "", + "parallel-test-types-list-as-string": None, }, id="Nothing should run if only non-important files changed", ), @@ -495,17 +535,21 @@ def test_expected_output_full_tests_needed( "tests/providers/google/file.py", ), { + "affected-providers-list-as-string": "amazon apache.beam apache.cassandra cncf.kubernetes " + "common.sql facebook google hashicorp microsoft.azure microsoft.mssql " + "mysql oracle postgres presto salesforce sftp ssh trino", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "needs-helm-tests": "false", "image-build": "true", "run-tests": "true", "docs-build": "true", + "docs-filter-list-as-string": "--package-filter apache-airflow --package-filter docker-stack", "full-tests-needed": "false", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", - "parallel-test-types": "Always", + "parallel-test-types-list-as-string": "Always", }, id="No Helm tests, No providers should run if only chart/providers changed in non-main", ), @@ -516,17 +560,22 @@ def test_expected_output_full_tests_needed( "tests/providers/google/file.py", ), { + "affected-providers-list-as-string": "amazon apache.beam apache.cassandra " + "cncf.kubernetes common.sql facebook google " + "hashicorp microsoft.azure microsoft.mssql mysql oracle postgres presto " + "salesforce sftp ssh trino", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", "docs-build": "true", + "docs-filter-list-as-string": "--package-filter apache-airflow --package-filter docker-stack", "full-tests-needed": "false", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", - "parallel-test-types": "Always CLI", + "parallel-test-types-list-as-string": "Always CLI", }, id="Only CLI tests and Kubernetes tests should run if cli/chart files changed in non-main branch", ), @@ -536,17 +585,19 @@ def test_expected_output_full_tests_needed( "tests/providers/google/file.py", ), { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", "docs-build": "true", + "docs-filter-list-as-string": "--package-filter apache-airflow --package-filter docker-stack", "full-tests-needed": "false", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", - "parallel-test-types": "Core Other WWW API Always CLI", + "parallel-test-types-list-as-string": "Core Other WWW API Always CLI", }, id="All tests except Providers should run if core file changed in non-main branch", ), @@ -572,30 +623,34 @@ def test_expected_output_pull_request_v2_3( pytest.param( ("INTHEWILD.md",), { + "affected-providers-list-as-string": None, "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "image-build": "false", "needs-helm-tests": "false", "run-tests": "false", "docs-build": "false", + "docs-filter-list-as-string": None, "upgrade-to-newer-dependencies": "false", - "skip-provider-tests": "false", - "parallel-test-types": "", + "skip-provider-tests": "true", + "parallel-test-types-list-as-string": None, }, id="Nothing should run if only non-important files changed", ), pytest.param( ("tests/system/any_file.py",), { + "affected-providers-list-as-string": None, "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", "docs-build": "true", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "upgrade-to-newer-dependencies": "false", - "skip-provider-tests": "false", - "parallel-test-types": "Always", + "skip-provider-tests": "true", + "parallel-test-types-list-as-string": "Always", }, id="Only Always and docs build should run if only system tests changed", ), @@ -606,16 +661,39 @@ def test_expected_output_pull_request_v2_3( "tests/providers/google/file.py", ), { + "affected-providers-list-as-string": "amazon apache.beam apache.cassandra " + "cncf.kubernetes common.sql " + "facebook google hashicorp microsoft.azure microsoft.mssql mysql " + "oracle postgres presto salesforce sftp ssh trino", "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", "docs-build": "true", + "docs-filter-list-as-string": "--package-filter apache-airflow --package-filter helm-chart " + "--package-filter apache-airflow-providers-amazon " + "--package-filter apache-airflow-providers-apache-beam " + "--package-filter apache-airflow-providers-apache-cassandra " + "--package-filter apache-airflow-providers-cncf-kubernetes " + "--package-filter apache-airflow-providers-common-sql " + "--package-filter apache-airflow-providers-facebook " + "--package-filter apache-airflow-providers-google " + "--package-filter apache-airflow-providers-hashicorp " + "--package-filter apache-airflow-providers-microsoft-azure " + "--package-filter apache-airflow-providers-microsoft-mssql " + "--package-filter apache-airflow-providers-mysql " + "--package-filter apache-airflow-providers-oracle " + "--package-filter apache-airflow-providers-postgres " + "--package-filter apache-airflow-providers-presto " + "--package-filter apache-airflow-providers-salesforce " + "--package-filter apache-airflow-providers-sftp " + "--package-filter apache-airflow-providers-ssh " + "--package-filter apache-airflow-providers-trino", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "false", - "parallel-test-types": "Providers[amazon] Always CLI " + "parallel-test-types-list-as-string": "Providers[amazon] Always CLI " "Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.sql,facebook," "hashicorp,microsoft.azure,microsoft.mssql,mysql,oracle,postgres,presto," "salesforce,sftp,ssh,trino] Providers[google]", @@ -624,23 +702,65 @@ def test_expected_output_pull_request_v2_3( ), pytest.param( ( - "airflow/file.py", - "tests/providers/google/file.py", + "airflow/cli/file.py", + "airflow/www/file.py", + "airflow/api/file.py", ), { + "affected-providers-list-as-string": None, + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "docs-build": "true", + "docs-filter-list-as-string": "--package-filter apache-airflow", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "skip-provider-tests": "true", + "parallel-test-types-list-as-string": "WWW API Always CLI", + }, + id="No providers tests should run if only CLI/API/WWW file changed", + ), + pytest.param( + ("airflow/models/test.py",), + { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "docs-build": "true", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "skip-provider-tests": "false", + "parallel-test-types-list-as-string": "Core Providers[-amazon,google] Other " + "Providers[amazon] WWW " + "API Always CLI Providers[google]", + }, + id="Tests for all providers should run if model file changed", + ), + pytest.param( + ("airflow/file.py",), + { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7']", "all-python-versions-list-as-string": "3.7", "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", "docs-build": "true", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "false", - "parallel-test-types": "Core Providers[-amazon,google] Other Providers[amazon] WWW " + "parallel-test-types-list-as-string": "Core Providers[-amazon,google] Other " + "Providers[amazon] WWW " "API Always CLI Providers[google]", }, - id="All tests should run if core file changed", + id="Tests for all providers should run if any other than API/WWW/CLI file changed.", ), ], ) @@ -666,15 +786,17 @@ def test_expected_output_pull_request_target( (), "main", { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", "docs-build": "true", - "docs-filter": "", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "upgrade-to-newer-dependencies": "true", - "parallel-test-types": "Core Providers[-amazon,google] Other Providers[amazon] WWW " + "parallel-test-types-list-as-string": "Core Providers[-amazon,google] Other " + "Providers[amazon] WWW " "API Always CLI Providers[google]", }, id="All tests run on push even if unimportant file changed", @@ -684,15 +806,16 @@ def test_expected_output_pull_request_target( (), "v2-3-stable", { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", "docs-build": "true", - "docs-filter": "--package-filter apache-airflow --package-filter docker-stack", + "docs-filter-list-as-string": "--package-filter apache-airflow --package-filter docker-stack", "upgrade-to-newer-dependencies": "true", - "parallel-test-types": "Core Other WWW API Always CLI", + "parallel-test-types-list-as-string": "Core Other WWW API Always CLI", }, id="All tests except Providers and Helm run on push" " even if unimportant file changed in non-main branch", @@ -702,15 +825,17 @@ def test_expected_output_pull_request_target( (), "main", { + "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", "docs-build": "true", - "docs-filter": "", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "upgrade-to-newer-dependencies": "true", - "parallel-test-types": "Core Providers[-amazon,google] Other Providers[amazon] WWW " + "parallel-test-types-list-as-string": "Core Providers[-amazon,google] Other " + "Providers[amazon] WWW " "API Always CLI Providers[google]", }, id="All tests run on push if core file changed", @@ -762,7 +887,8 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): "upgrade-to-newer-dependencies": "true" if github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE] else "false", - "parallel-test-types": "Core Providers[-amazon,google] Other Providers[amazon] WWW " + "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " + "Other Providers[amazon] WWW " "API Always CLI Providers[google]", }, str(stderr), @@ -826,7 +952,7 @@ def test_upgrade_to_newer_dependencies(files: tuple[str, ...], expected_outputs: pytest.param( ("docs/apache-airflow-providers-google/docs.rst",), { - "docs-filter": "--package-filter apache-airflow-providers-amazon " + "docs-filter-list-as-string": "--package-filter apache-airflow-providers-amazon " "--package-filter apache-airflow-providers-apache-beam " "--package-filter apache-airflow-providers-apache-cassandra " "--package-filter apache-airflow-providers-cncf-kubernetes " @@ -850,7 +976,7 @@ def test_upgrade_to_newer_dependencies(files: tuple[str, ...], expected_outputs: pytest.param( ("airflow/providers/common/sql/common_sql_python.py",), { - "docs-filter": "--package-filter apache-airflow " + "docs-filter-list-as-string": "--package-filter apache-airflow " "--package-filter apache-airflow-providers-amazon " "--package-filter apache-airflow-providers-apache-drill " "--package-filter apache-airflow-providers-apache-druid " @@ -881,7 +1007,7 @@ def test_upgrade_to_newer_dependencies(files: tuple[str, ...], expected_outputs: pytest.param( ("docs/apache-airflow-providers-airbyte/docs.rst",), { - "docs-filter": "--package-filter apache-airflow-providers-airbyte " + "docs-filter-list-as-string": "--package-filter apache-airflow-providers-airbyte " "--package-filter apache-airflow-providers-http", }, id="Airbyte provider docs changed", @@ -889,7 +1015,7 @@ def test_upgrade_to_newer_dependencies(files: tuple[str, ...], expected_outputs: pytest.param( ("airflow/providers/celery/file.py",), { - "docs-filter": "--package-filter apache-airflow " + "docs-filter-list-as-string": "--package-filter apache-airflow " "--package-filter apache-airflow-providers-celery", }, id="Celery python files changed", @@ -897,40 +1023,40 @@ def test_upgrade_to_newer_dependencies(files: tuple[str, ...], expected_outputs: pytest.param( ("docs/conf.py",), { - "docs-filter": "", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, }, id="Docs conf.py changed", ), pytest.param( ("airflow/test.py",), { - "docs-filter": "--package-filter apache-airflow", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, }, - id="Core files changed. No provider docs to build", + id="Core files changed. All provider docs should also be built", ), pytest.param( ("docs/docker-stack/test.rst",), - {"docs-filter": "--package-filter docker-stack"}, + {"docs-filter-list-as-string": "--package-filter docker-stack"}, id="Docker stack files changed. No provider docs to build", ), pytest.param( ("airflow/test.py", "chart/airflow/values.yaml"), { - "docs-filter": "--package-filter apache-airflow --package-filter helm-chart", + "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, }, - id="Core files and helm chart files changed. No provider docs to build", + id="Core files and helm chart files changed. All provider docs should be built", ), pytest.param( ("chart/airflow/values.yaml",), { - "docs-filter": "--package-filter helm-chart", + "docs-filter-list-as-string": "--package-filter helm-chart", }, id="Helm chart files changed. No provider, airflow docs to build", ), pytest.param( ("docs/helm-chart/airflow/values.yaml",), { - "docs-filter": "--package-filter helm-chart", + "docs-filter-list-as-string": "--package-filter helm-chart", }, id="Docs helm chart files changed. No provider, airflow docs to build", ),