Cleanup installed packages when running provider compatibility tests

.github/workflows/check-providers.yml

@@ -28,6 +28,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Tag to set for the image"
         required: true
         type: string
+      canary-run:
+        description: "Whether this is a canary run"
+        required: true
+        type: string
       default-python-version:
         description: "Which version of python should be used by default"
         required: true
@@ -209,6 +213,7 @@ jobs:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       VERBOSE: "true"
+      CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}"
     if: inputs.skip-provider-tests != 'true'
     steps:
       - name: "Cleanup repo"

.github/workflows/ci.yml

@@ -319,6 +319,7 @@ jobs:
     with:
       runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
+      canary-run: ${{ needs.build-info.outputs.canary-run }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }}

Dockerfile.ci

@@ -997,15 +997,23 @@ function determine_airflow_to_use() {
             echo
             exit 0
         fi
+        if [[ ${CLEAN_AIRFLOW_INSTALLATION=} == "true" ]]; then
+            echo
+            echo "${COLOR_BLUE}Uninstalling all packages first${COLOR_RESET}"
+            echo
+            pip freeze | grep -ve "^-e" | grep -ve "^#" | grep -ve "^uv" | xargs pip uninstall -y --root-user-action ignore
+            # Now install rich ad click first to use the installation script
+            uv pip install rich rich-click click --python "/usr/local/bin/python" \
+                --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
+        fi
         python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py"
+        echo
+        echo "${COLOR_BLUE}Reinstalling all development dependencies${COLOR_RESET}"
+        echo
+        python "${IN_CONTAINER_DIR}/install_devel_deps.py" \
+           --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
         # Some packages might leave legacy typing module which causes test issues
         pip uninstall -y typing || true
-        # Upgrade pytest and pytest extensions to latest version if they have been accidentally
-        # downgraded by constraints
-        pip install --upgrade pytest pytest aiofiles aioresponses pytest-asyncio pytest-custom-exit-code \
-           pytest-icdiff pytest-instafail pytest-mock pytest-rerunfailures pytest-timeouts \
-           pytest-xdist pytest requests_mock time-machine \
-           --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
     fi
 
     if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then

contributing-docs/testing/unit_tests.rst

@@ -1226,6 +1226,12 @@ Running provider compatibility tests in CI
 In CI those tests are run in a slightly more complex way because we want to run them against the build
 provider packages, rather than mounted from sources.
 
+In case of canary runs we add ``--clean-airflow-installation`` flag that removes all packages before
+installing older airflow version, and then installs development dependencies
+from latest airflow - in order to avoid case where a provider depends on a new dependency added in latest
+version of Airflow. This clean removal and re-installation takes quite some time though and in order to
+speed up the tests in regular PRs we only do that in the canary runs.
+
 The exact way CI tests are run can be reproduced locally building providers from selected tag/commit and
 using them to install and run tests against the selected airflow version.
 
@@ -1262,6 +1268,14 @@ Herr id how to reproduce it.
   breeze shell --use-packages-from-dist --package-format wheel --use-airflow-version 2.9.1  \
    --install-airflow-with-constraints --providers-skip-constraints --mount-sources tests
 
+In case you want to reproduce canary run, you need to add ``--clean-airflow-installation`` flag:
+
+.. code-block:: bash
+
+  breeze shell --use-packages-from-dist --package-format wheel --use-airflow-version 2.9.1  \
+   --install-airflow-with-constraints --providers-skip-constraints --mount-sources tests --clean-airflow-installation
+
+
 6. You can then run tests as usual:
 
 .. code-block:: bash

dev/breeze/doc/images/output_release-management_install-provider-packages.txt

@@ -1 +1 @@
-7d8e9b3e73eda99894455d573ea940d7
+47c9f57fabfb8bae746afdf3eedc65b0

dev/breeze/doc/images/output_release-management_verify-provider-packages.txt

@@ -1 +1 @@
-f5c686ce50cae1ea4647b0f2202f731b
+c092d6e71312d96e28f98f537fd88796

dev/breeze/doc/images/output_shell.txt

@@ -1 +1 @@
-56116218e3de632fa766f9c7368315a8
+23410acb975d790d4f6bbd037cb54dff

dev/breeze/doc/images/output_start-airflow.txt

@@ -1 +1 @@
-929ca9ec2fc8c229c302b161f610059b
+c09c2c71d596f531accec1c41eee288e