Merge remote-tracking branch 'origin/main' into bugfix/fix-index-urls-when-venv-is-created-with-uv

Author: wolfdn

.github/actions/breeze/action.yml

@@ -21,7 +21,7 @@ description: 'Sets up Python and Breeze'
 inputs:
   python-version:
     description: 'Python version to use'
-    default: "3.9"
+    default: "3.10"
   use-uv:
     description: 'Whether to use uv tool'
     required: true
@@ -41,6 +41,8 @@ runs:
     - name: "Install Breeze"
       shell: bash
       run: ./scripts/ci/install_breeze.sh
+      env:
+        PYTHON_VERSION: "${{ inputs.python-version }}"
     - name: "Free space"
       shell: bash
       run: breeze ci free-space

.github/actions/install-pre-commit/action.yml

@@ -21,10 +21,10 @@ description: 'Installs pre-commit and related packages'
 inputs:
   python-version:
     description: 'Python version to use'
-    default: "3.9"
+    default: "3.10"
   uv-version:
     description: 'uv version to use'
-    default: "0.7.14"  # Keep this comment to allow automatic replacement of uv version
+    default: "0.7.16"  # Keep this comment to allow automatic replacement of uv version
   pre-commit-version:
     description: 'pre-commit version to use'
     default: "4.2.0"  # Keep this comment to allow automatic replacement of pre-commit version

.github/actions/prepare_all_ci_images/action.yml

@@ -34,14 +34,9 @@ runs:
     # TODO: Currently we cannot loop through the list of python versions and have dynamic list of
     #       tasks. Instead we hardcode all possible python versions and they - but
     #       this should be implemented in stash action as list of keys to download.
-    #       That includes 3.8 - 3.12 as we are backporting it to v2-10-test branch
+    #       That includes 3.9 - 3.12 as we are backporting it to v3-0-test branch
     #       This is captured in https://github.com/apache/airflow/issues/45268
-    - name: "Restore CI docker image ${{ inputs.platform }}:3.8"
-      uses: ./.github/actions/prepare_single_ci_image
-      with:
-        platform: ${{ inputs.platform }}
-        python: "3.8"
-        python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
+    #       So we actually need 3.9 even if 3.9 support on main is dropped!
     - name: "Restore CI docker image ${{ inputs.platform }}:3.9"
       uses: ./.github/actions/prepare_single_ci_image
       with:

.github/workflows/airflow-distributions-tests.yml

@@ -61,11 +61,16 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether local venv should be used for tests (true/false)"
         required: true
         type: string
+      test-timeout:
+        required: false
+        type: number
+        default: 60
+
 permissions:
   contents: read
 jobs:
   distributions-tests:
-    timeout-minutes: 80
+    timeout-minutes: ${{ fromJSON(inputs.test-timeout) }}
     name: ${{ inputs.distribution-name }}:P${{ matrix.python-version }} tests
     runs-on: ${{ fromJSON(inputs.runners) }}
     strategy:

.github/workflows/basic-tests.yml

@@ -223,7 +223,7 @@ jobs:
           VERBOSE: "false"
           SKIP_BREEZE_PRE_COMMITS: "true"
           SKIP: ${{ inputs.skip-pre-commits }}
-          COLUMNS: "250"
+          COLUMNS: "202"
 
   test-git-clone-on-windows:
     timeout-minutes: 5
@@ -317,7 +317,6 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       GITHUB_USERNAME: ${{ github.actor }}
       VERBOSE: "true"
-    if: inputs.canary-run == 'true'
     steps:
       - name: "Cleanup repo"
         shell: bash

.github/workflows/ci-amd.yml

@@ -186,7 +186,7 @@ jobs:
       - name: Install pre-commit, uv, and pre-commit-uv
         shell: bash
         env:
-          UV_VERSION: "0.7.14"  # Keep this comment to allow automatic replacement of uv version
+          UV_VERSION: "0.7.16"  # Keep this comment to allow automatic replacement of uv version
           PRE_COMMIT_VERSION: "4.2.0"  # Keep this comment to allow automatic replacement of pre-commit version
           PRE_COMMIT_UV_VERSION: "4.1.4"  # Keep this comment to allow automatic replacement of pre-commit-uv version
         run: |
@@ -248,7 +248,6 @@ jobs:
       contents: read
       packages: write
       id-token: write
-    if: needs.build-info.outputs.canary-run == 'true'
     with:
       runners: ${{ needs.build-info.outputs.amd-runners }}
       platform: "linux/amd64"
@@ -275,6 +274,8 @@ jobs:
       runners: ${{ needs.build-info.outputs.amd-runners }}
       platform: "linux/amd64"
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+      python-versions: ${{ needs.build-info.outputs.python-versions }}
+      generate-pypi-constraints: "true"
       # generate no providers constraints only in canary builds - they take quite some time to generate
       # they are not needed for regular builds, they are only needed to update constraints in canaries
       generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }}
@@ -784,6 +785,7 @@ jobs:
       distribution-cmd-format: "prepare-task-sdk-distributions"
       test-type: "task-sdk-tests"
       use-local-venv: 'false'
+      test-timeout: 20
     if: >
       ( needs.build-info.outputs.run-task-sdk-tests == 'true' ||
       needs.build-info.outputs.run-tests == 'true' &&
@@ -819,12 +821,19 @@ jobs:
           go-version: 1.24
           cache-dependency-path: go-sdk/go.sum
 
+      # keep this in sync with go.mod in go-sdk/
+      - name: Setup Gotestsum
+        shell: bash
+        run: |
+          go install gotest.tools/gotestsum@ddd0b05a6878e2e8257a2abe6e7df66cebc53d0e  # v1.12.3
+          gotestsum --version
+
       - name: "Cleanup dist files"
         run: rm -fv ./dist/*
 
       - name: Run Go tests
         working-directory: ./go-sdk
-        run: go test -v ./...
+        run: gotestsum --format testname ./...
 
   tests-airflow-ctl:
     name: "Airflow CTL tests"
@@ -844,6 +853,7 @@ jobs:
       distribution-cmd-format: "prepare-airflow-ctl-distributions"
       test-type: "airflow-ctl-tests"
       use-local-venv: 'true'
+      test-timeout: 20
     if: >
       ( needs.build-info.outputs.run-airflow-ctl-tests == 'true' ||
       needs.build-info.outputs.run-tests == 'true' &&

.github/workflows/ci-arm.yml

@@ -210,7 +210,6 @@ jobs:
       contents: read
       packages: write
       id-token: write
-    if: needs.build-info.outputs.canary-run == 'true'
     with:
       runners: ${{ needs.build-info.outputs.arm-runners }}
       platform: "linux/arm64"
@@ -237,13 +236,14 @@ jobs:
       runners: ${{ needs.build-info.outputs.arm-runners }}
       platform: "linux/arm64"
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+      python-versions: ${{ needs.build-info.outputs.python-versions }}
+      generate-pypi-constraints: "true"
       # generate no providers constraints only in canary builds - they take quite some time to generate
       # they are not needed for regular builds, they are only needed to update constraints in canaries
       generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
 
-
   providers:
     name: "provider distributions tests"
     uses: ./.github/workflows/test-providers.yml

.github/workflows/ci-image-checks.yml

@@ -193,7 +193,7 @@ jobs:
         env:
           VERBOSE: "false"
           SKIP: ${{ inputs.skip-pre-commits }}
-          COLUMNS: "250"
+          COLUMNS: "202"
           SKIP_GROUP_OUTPUT: "true"
           DEFAULT_BRANCH: ${{ inputs.branch }}
           RUFF_FORMAT: "github"
@@ -235,7 +235,7 @@ jobs:
         run: pre-commit run --color always --verbose --hook-stage manual "$MYPY_CHECK" --all-files
         env:
           VERBOSE: "false"
-          COLUMNS: "250"
+          COLUMNS: "202"
           SKIP_GROUP_OUTPUT: "true"
           DEFAULT_BRANCH: ${{ inputs.branch }}
           RUFF_FORMAT: "github"

.github/workflows/finalize-tests.yml

@@ -118,7 +118,7 @@ jobs:
       - name: "Download constraints from the constraints generated by build CI image"
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
         with:
-          name: constraints
+          pattern: constraints-*
           path: ./files
       - name: "Diff in constraints for Python: ${{ inputs.python-versions-list-as-string }}"
         run: ./scripts/ci/constraints/ci_diff_constraints.sh
@@ -132,6 +132,34 @@ jobs:
         run:
           git push
 
+  dependency-upgrade-summary:
+    runs-on: ${{ fromJSON(inputs.runners) }}
+    needs: [update-constraints]
+    if: inputs.upgrade-to-newer-dependencies == 'true' && inputs.platform == 'linux/amd64'
+    name: "Dependency upgrade summary"
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          persist-credentials: false
+      - name: "Prepare breeze & CI image: ${{ matrix.python-version }}"
+        uses: ./.github/actions/prepare_breeze_and_image
+        with:
+          platform: ${{ inputs.platform }}
+          python: ${{ inputs.default-python-version }}
+          use-uv: ${{ inputs.use-uv }}
+      - name: "Dependency upgrade summary"
+        shell: bash
+        run: >
+          breeze run uv run /opt/airflow/dev/constraints-updated-version-check.py --explain-why
+        env:
+          PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+
   push-buildx-cache-to-github-registry:
     name: Push Regular Image Cache ${{ inputs.platform }}
     needs: [update-constraints]

.github/workflows/generate-constraints.yml

@@ -32,10 +32,18 @@ on:  # yamllint disable-line rule:truthy
         description: "Stringified array of all Python versions to test - separated by spaces."
         required: true
         type: string
+      python-versions:
+        description: "JSON-formatted array of Python versions to generate constraints for"
+        required: true
+        type: string
       generate-no-providers-constraints:
         description: "Whether to generate constraints without providers (true/false)"
         required: true
         type: string
+      generate-pypi-constraints:
+        description: "Whether to generate PyPI constraints (true/false)"
+        required: true
+        type: string
       debug-resources:
         description: "Whether to run in debug mode (true/false)"
         required: true
@@ -45,19 +53,22 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
 jobs:
-  generate-constraints:
+  generate-constraints-matrix:
     permissions:
       contents: read
     timeout-minutes: 70
-    name: Generate constraints ${{ inputs.python-versions-list-as-string }}
+    name: Generate constraints for ${{ matrix.python-version }} on ${{ inputs.platform }}
     runs-on: ${{ fromJSON(inputs.runners) }}
+    strategy:
+      matrix:
+        python-version: ${{ fromJson(inputs.python-versions) }}
     env:
       DEBUG_RESOURCES: ${{ inputs.debug-resources }}
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       GITHUB_USERNAME: ${{ github.actor }}
       INCLUDE_SUCCESS_OUTPUTS: "true"
-      PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }}
+      PYTHON_VERSION: ${{ matrix.python-version }}
       VERBOSE: "true"
     steps:
       - name: "Cleanup repo"
@@ -67,72 +78,61 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
         with:
           persist-credentials: false
-      - name: "Prepare and cleanup runner"
-        run: ./scripts/ci/prepare_and_cleanup_runner.sh
-        shell: bash
-      - name: "Install Breeze"
-        uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
-        id: breeze
-      - name: "Prepare all CI images: ${{ inputs.python-versions-list-as-string}}"
-        uses: ./.github/actions/prepare_all_ci_images
+      - name: "Prepare breeze & CI image: ${{ matrix.python-version }}"
+        uses: ./.github/actions/prepare_breeze_and_image
         with:
           platform: ${{ inputs.platform }}
-          python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
-          docker-volume-location: ""  # TODO(jscheffl): Understand why it fails here and fix it
-      - name: "Verify all CI images ${{ inputs.python-versions-list-as-string }}"
-        run: breeze ci-image verify --run-in-parallel
+          python: ${{ matrix.python-version }}
+          use-uv: ${{ inputs.use-uv }}
       - name: "Source constraints"
         shell: bash
         run: >
-          breeze release-management generate-constraints --run-in-parallel
+          breeze release-management generate-constraints
           --airflow-constraints-mode constraints-source-providers --answer yes
+          --python "${PYTHON_VERSION}"
       - name: "No providers constraints"
         shell: bash
         timeout-minutes: 25
         run: >
-          breeze release-management generate-constraints --run-in-parallel
-          --airflow-constraints-mode constraints-no-providers --answer yes --parallelism 3
-        # The no providers constraints are only needed when we want to update constraints (in canary builds)
-        # They slow down the start of PROD image builds so we want to only run them when needed.
+          breeze release-management generate-constraints
+          --airflow-constraints-mode constraints-no-providers --answer yes
+          --python "${PYTHON_VERSION}"
         if: inputs.generate-no-providers-constraints == 'true'
       - name: "Prepare updated provider distributions"
-        # In case of provider distributions which are not yet released, we build them from sources
         shell: bash
         run: >
-          breeze release-management prepare-provider-distributions --include-not-ready-providers
-          --distribution-format wheel
+          breeze release-management prepare-provider-distributions
+          --include-not-ready-providers --distribution-format wheel
+        if: inputs.generate-pypi-constraints == 'true'
       - name: "Prepare airflow distributions"
         shell: bash
         run: >
           breeze release-management prepare-airflow-distributions --distribution-format wheel
+        if: inputs.generate-pypi-constraints == 'true'
       - name: "Prepare task-sdk distribution"
         shell: bash
         run: >
           breeze release-management prepare-task-sdk-distributions --distribution-format wheel
+        if: inputs.generate-pypi-constraints == 'true'
       - name: "PyPI constraints"
         shell: bash
         timeout-minutes: 25
         run: |
-          for PYTHON in $PYTHON_VERSIONS; do
-            breeze release-management generate-constraints --airflow-constraints-mode constraints \
-            --answer yes --python "${PYTHON}"
-          done
-      - name: "Dependency upgrade summary"
-        shell: bash
-        env:
-          PYTHON_VERSIONS: ${{ env.PYTHON_VERSIONS }}
-        run: |
-          for PYTHON_VERSION in $PYTHON_VERSIONS; do
-            echo "Summarizing Python $PYTHON_VERSION"
-            cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true
-            df -H
-          done
+          breeze release-management generate-constraints --airflow-constraints-mode constraints \
+            --answer yes --python "${PYTHON_VERSION}"
+        if: inputs.generate-pypi-constraints == 'true'
       - name: "Upload constraint artifacts"
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
         with:
-          name: constraints
-          path: ./files/constraints-*/constraints-*.txt
+          name: constraints-${{ matrix.python-version }}
+          path: ./files/constraints-${{ matrix.python-version }}/constraints-*.txt
           retention-days: 7
           if-no-files-found: error
+      - name: "Dependency upgrade summary"
+        shell: bash
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+        run: |
+          echo "Summarizing Python $PYTHON_VERSION"
+          cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true
+          df -H