From d85f6cf7803d851408d6338442f9cbec456714b8 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 14 Aug 2024 20:36:07 +0200 Subject: [PATCH] Cleanup installed packages when running provider compatibility tests When running tests for providers with old Airflow versions, we have to make sure to uninstall all packages before installing old Airflow versions - because some of the packages can be installed by new Airflow version and they are missing in the old Airflow version. This has already happened in #41402 with methodtools. This PR adds `--clean-airflow-installation` flag to relevant breeze commands that install other airflow version. This is quite a bit slower as it requires to uninstall and reinstall packages so we do not set it by default. --- .github/workflows/check-providers.yml | 5 + .github/workflows/ci.yml | 1 + Dockerfile.ci | 20 ++- contributing-docs/testing/unit_tests.rst | 14 ++ ...e-management_install-provider-packages.svg | 104 ++++++------ ...e-management_install-provider-packages.txt | 2 +- ...se-management_verify-provider-packages.svg | 102 ++++++------ ...se-management_verify-provider-packages.txt | 2 +- dev/breeze/doc/images/output_shell.svg | 150 +++++++++--------- dev/breeze/doc/images/output_shell.txt | 2 +- .../doc/images/output_start-airflow.svg | 146 +++++++++-------- .../doc/images/output_start-airflow.txt | 2 +- .../doc/images/output_testing_db-tests.svg | 104 ++++++------ .../doc/images/output_testing_db-tests.txt | 2 +- .../images/output_testing_non-db-tests.svg | 104 ++++++------ .../images/output_testing_non-db-tests.txt | 2 +- .../doc/images/output_testing_tests.svg | 104 ++++++------ .../doc/images/output_testing_tests.txt | 2 +- .../airflow_breeze/commands/common_options.py | 6 + .../commands/developer_commands.py | 7 + .../commands/developer_commands_config.py | 2 + .../commands/release_management_commands.py | 7 + .../release_management_commands_config.py | 2 + .../commands/testing_commands.py | 6 + .../commands/testing_commands_config.py | 3 + .../src/airflow_breeze/params/shell_params.py | 2 + hatch_build.py | 1 + pyproject.toml | 1 + scripts/docker/entrypoint_ci.sh | 20 ++- .../install_airflow_and_providers.py | 13 +- scripts/in_container/install_devel_deps.py | 77 +++++++++ .../test_aws_security_manager_override.py | 2 + .../celery/cli/test_celery_command.py | 8 + .../celery/executors/test_celery_executor.py | 5 +- 34 files changed, 627 insertions(+), 403 deletions(-) create mode 100755 scripts/in_container/install_devel_deps.py diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml index 622a67fea97a1..e89d4a81faaca 100644 --- a/.github/workflows/check-providers.yml +++ b/.github/workflows/check-providers.yml @@ -28,6 +28,10 @@ on: # yamllint disable-line rule:truthy description: "Tag to set for the image" required: true type: string + canary-run: + description: "Whether this is a canary run" + required: true + type: string default-python-version: description: "Which version of python should be used by default" required: true @@ -209,6 +213,7 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" VERSION_SUFFIX_FOR_PYPI: "dev0" VERBOSE: "true" + CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}" if: inputs.skip-provider-tests != 'true' steps: - name: "Cleanup repo" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e1db2a14edfc..fda01f71a5c3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -319,6 +319,7 @@ jobs: with: runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} image-tag: ${{ needs.build-info.outputs.image-tag }} + canary-run: ${{ needs.build-info.outputs.canary-run }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }} diff --git a/Dockerfile.ci b/Dockerfile.ci index 06a7343c82e26..0561761d71c1b 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -997,15 +997,23 @@ function determine_airflow_to_use() { echo exit 0 fi + if [[ ${CLEAN_AIRFLOW_INSTALLATION=} == "true" ]]; then + echo + echo "${COLOR_BLUE}Uninstalling all packages first${COLOR_RESET}" + echo + pip freeze | grep -ve "^-e" | grep -ve "^#" | grep -ve "^uv" | xargs pip uninstall -y --root-user-action ignore + # Now install rich ad click first to use the installation script + uv pip install rich rich-click click --python "/usr/local/bin/python" \ + --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt + fi python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py" + echo + echo "${COLOR_BLUE}Reinstalling all development dependencies${COLOR_RESET}" + echo + python "${IN_CONTAINER_DIR}/install_devel_deps.py" \ + --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt # Some packages might leave legacy typing module which causes test issues pip uninstall -y typing || true - # Upgrade pytest and pytest extensions to latest version if they have been accidentally - # downgraded by constraints - pip install --upgrade pytest pytest aiofiles aioresponses pytest-asyncio pytest-custom-exit-code \ - pytest-icdiff pytest-instafail pytest-mock pytest-rerunfailures pytest-timeouts \ - pytest-xdist pytest requests_mock time-machine \ - --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt fi if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index 6f43d901fca7b..8d27db19c2d5d 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -1226,6 +1226,12 @@ Running provider compatibility tests in CI In CI those tests are run in a slightly more complex way because we want to run them against the build provider packages, rather than mounted from sources. +In case of canary runs we add ``--clean-airflow-installation`` flag that removes all packages before +installing older airflow version, and then installs development dependencies +from latest airflow - in order to avoid case where a provider depends on a new dependency added in latest +version of Airflow. This clean removal and re-installation takes quite some time though and in order to +speed up the tests in regular PRs we only do that in the canary runs. + The exact way CI tests are run can be reproduced locally building providers from selected tag/commit and using them to install and run tests against the selected airflow version. @@ -1262,6 +1268,14 @@ Herr id how to reproduce it. breeze shell --use-packages-from-dist --package-format wheel --use-airflow-version 2.9.1 \ --install-airflow-with-constraints --providers-skip-constraints --mount-sources tests +In case you want to reproduce canary run, you need to add ``--clean-airflow-installation`` flag: + +.. code-block:: bash + + breeze shell --use-packages-from-dist --package-format wheel --use-airflow-version 2.9.1 \ + --install-airflow-with-constraints --providers-skip-constraints --mount-sources tests --clean-airflow-installation + + 6. You can then run tests as usual: .. code-block:: bash diff --git a/dev/breeze/doc/images/output_release-management_install-provider-packages.svg b/dev/breeze/doc/images/output_release-management_install-provider-packages.svg index 350a2d571f1dd..9610cac4f732d 100644 --- a/dev/breeze/doc/images/output_release-management_install-provider-packages.svg +++ b/dev/breeze/doc/images/output_release-management_install-provider-packages.svg @@ -1,4 +1,4 @@ - +