diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml index 622a67fea97a1..e89d4a81faaca 100644 --- a/.github/workflows/check-providers.yml +++ b/.github/workflows/check-providers.yml @@ -28,6 +28,10 @@ on: # yamllint disable-line rule:truthy description: "Tag to set for the image" required: true type: string + canary-run: + description: "Whether this is a canary run" + required: true + type: string default-python-version: description: "Which version of python should be used by default" required: true @@ -209,6 +213,7 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" VERSION_SUFFIX_FOR_PYPI: "dev0" VERBOSE: "true" + CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}" if: inputs.skip-provider-tests != 'true' steps: - name: "Cleanup repo" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e1db2a14edfc..fda01f71a5c3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -319,6 +319,7 @@ jobs: with: runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} image-tag: ${{ needs.build-info.outputs.image-tag }} + canary-run: ${{ needs.build-info.outputs.canary-run }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }} diff --git a/Dockerfile.ci b/Dockerfile.ci index 06a7343c82e26..0561761d71c1b 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -997,15 +997,23 @@ function determine_airflow_to_use() { echo exit 0 fi + if [[ ${CLEAN_AIRFLOW_INSTALLATION=} == "true" ]]; then + echo + echo "${COLOR_BLUE}Uninstalling all packages first${COLOR_RESET}" + echo + pip freeze | grep -ve "^-e" | grep -ve "^#" | grep -ve "^uv" | xargs pip uninstall -y --root-user-action ignore + # Now install rich ad click first to use the installation script + uv pip install rich rich-click click --python "/usr/local/bin/python" \ + --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt + fi python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py" + echo + echo "${COLOR_BLUE}Reinstalling all development dependencies${COLOR_RESET}" + echo + python "${IN_CONTAINER_DIR}/install_devel_deps.py" \ + --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt # Some packages might leave legacy typing module which causes test issues pip uninstall -y typing || true - # Upgrade pytest and pytest extensions to latest version if they have been accidentally - # downgraded by constraints - pip install --upgrade pytest pytest aiofiles aioresponses pytest-asyncio pytest-custom-exit-code \ - pytest-icdiff pytest-instafail pytest-mock pytest-rerunfailures pytest-timeouts \ - pytest-xdist pytest requests_mock time-machine \ - --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt fi if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index 6f43d901fca7b..8d27db19c2d5d 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -1226,6 +1226,12 @@ Running provider compatibility tests in CI In CI those tests are run in a slightly more complex way because we want to run them against the build provider packages, rather than mounted from sources. +In case of canary runs we add ``--clean-airflow-installation`` flag that removes all packages before +installing older airflow version, and then installs development dependencies +from latest airflow - in order to avoid case where a provider depends on a new dependency added in latest +version of Airflow. This clean removal and re-installation takes quite some time though and in order to +speed up the tests in regular PRs we only do that in the canary runs. + The exact way CI tests are run can be reproduced locally building providers from selected tag/commit and using them to install and run tests against the selected airflow version. @@ -1262,6 +1268,14 @@ Herr id how to reproduce it. breeze shell --use-packages-from-dist --package-format wheel --use-airflow-version 2.9.1 \ --install-airflow-with-constraints --providers-skip-constraints --mount-sources tests +In case you want to reproduce canary run, you need to add ``--clean-airflow-installation`` flag: + +.. code-block:: bash + + breeze shell --use-packages-from-dist --package-format wheel --use-airflow-version 2.9.1 \ + --install-airflow-with-constraints --providers-skip-constraints --mount-sources tests --clean-airflow-installation + + 6. You can then run tests as usual: .. code-block:: bash diff --git a/dev/breeze/doc/images/output_release-management_install-provider-packages.svg b/dev/breeze/doc/images/output_release-management_install-provider-packages.svg index 350a2d571f1dd..9610cac4f732d 100644 --- a/dev/breeze/doc/images/output_release-management_install-provider-packages.svg +++ b/dev/breeze/doc/images/output_release-management_install-provider-packages.svg @@ -1,4 +1,4 @@ - +