diff --git a/.dockerignore b/.dockerignore index edce6e9e78aef..c7fb2e60c7bb1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -47,6 +47,7 @@ !helm-tests !kubernetes-tests !task-sdk-tests +!airflow-ctl-tests !shared/ # Add scripts so that we can use them inside the container diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index f04a2ef0ea150..789490abf0388 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -25,15 +25,15 @@ body: the latest release or main to see if the issue is fixed before reporting it. multiple: false options: - - "3.0.6" + - "3.1.2" - "2.11.0" - "main (development)" - - "Other Airflow 2 version (please specify below)" + - "Other Airflow 2/3 version (please specify below)" validations: required: true - type: input attributes: - label: If "Other Airflow 2 version" selected, which one? + label: If "Other Airflow 2/3 version" selected, which one? # yamllint disable rule:line-length description: > On what 2.X version of Airflow are you currently experiencing the issue? Remember, you are encouraged to diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml index 7b583d8a45a36..21d63814d15fd 100644 --- a/.github/actions/breeze/action.yml +++ b/.github/actions/breeze/action.yml @@ -24,7 +24,7 @@ inputs: default: "3.10" uv-version: description: 'uv version to use' - default: "0.9.4" # Keep this comment to allow automatic replacement of uv version + default: "0.9.7" # Keep this comment to allow automatic replacement of uv version outputs: host-python-version: description: Python version used in host diff --git a/.github/actions/install-prek/action.yml b/.github/actions/install-prek/action.yml index 10f80d14959e0..85dbef1b3919d 100644 --- a/.github/actions/install-prek/action.yml +++ b/.github/actions/install-prek/action.yml @@ -24,10 +24,10 @@ inputs: default: "3.10" uv-version: description: 'uv version to use' - default: "0.9.4" # Keep this comment to allow automatic replacement of uv version + default: "0.9.7" # Keep this comment to allow automatic replacement of uv version prek-version: description: 'prek version to use' - default: "0.2.10" # Keep this comment to allow automatic replacement of prek version + default: "0.2.12" # Keep this comment to allow automatic replacement of prek version save-cache: description: "Whether to save prek cache" required: true diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 714761bceb3cc..c7d4bde61c8db 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -191,3 +191,34 @@ jobs: id: breeze - name: "Run Task SDK integration tests" run: breeze testing task-sdk-integration-tests + + airflow-ctl-integration-tests: + timeout-minutes: 60 + name: "Airflow CTL integration tests with PROD image" + runs-on: ${{ fromJSON(inputs.runners) }} + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 2 + persist-credentials: false + - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ inputs.platform }} + image-type: "prod" + python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} + use-uv: ${{ inputs.use-uv }} + make-mnt-writeable-and-cleanup: true + id: breeze + - name: "Run airflowctl integration tests" + run: breeze testing airflow-ctl-integration-tests diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 1fafc6e59379c..65943bd47f94d 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -66,7 +66,7 @@ on: # yamllint disable-line rule:truthy type: string uv-version: description: 'uv version to use' - default: "0.9.4" # Keep this comment to allow automatic replacement of uv version + default: "0.9.7" # Keep this comment to allow automatic replacement of uv version type: string platform: description: 'Platform for the build - linux/amd64 or linux/arm64' diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd-arm.yml similarity index 89% rename from .github/workflows/ci-amd.yml rename to .github/workflows/ci-amd-arm.yml index c5bff2395b03f..f42d4397a826b 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd-arm.yml @@ -16,10 +16,10 @@ # under the License. # --- -name: Tests AMD +name: Tests on: # yamllint disable-line rule:truthy schedule: - - cron: '28 1,7,13,19 * * *' + - cron: '28 1,3,7,9,13,15,19,21 * * *' push: branches: - v[0-9]+-[0-9]+-test @@ -43,7 +43,7 @@ env: VERBOSE: "true" concurrency: - group: ci-amd-${{ github.event.pull_request.number || github.ref }} + group: ci-amd-arm-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: @@ -57,7 +57,6 @@ jobs: outputs: all-python-versions-list-as-string: >- ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} - amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} canary-run: ${{ steps.source-run-info.outputs.canary-run }} @@ -94,6 +93,7 @@ jobs: mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} + platform: ${{ steps.selective-checks.outputs.platform }} postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} @@ -116,6 +116,7 @@ jobs: run-mypy: ${{ steps.selective-checks.outputs.run-mypy }} run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} + runner-type: ${{ steps.selective-checks.outputs.runner-type }} run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }} run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} @@ -165,10 +166,30 @@ jobs: env: PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} GITHUB_CONTEXT: ${{ toJson(github) }} + + print-platform-arm: + name: "Platform: ARM" + needs: [build-info] + runs-on: ["ubuntu-22.04"] + if: needs.build-info.outputs.platform == 'linux/arm64' + steps: + - name: "Print architecture" + run: "echo '## Architecture: ARM' >> $GITHUB_STEP_SUMMARY" + + print-platform-amd: + name: "Platform: AMD" + needs: [build-info] + runs-on: ["ubuntu-22.04"] + if: needs.build-info.outputs.platform == 'linux/amd64' + steps: + - name: "Print architecture" + run: "echo '## Architecture: AMD' >> $GITHUB_STEP_SUMMARY" + run-pin-versions-hook: - name: "Run pin-versions hook" + name: "Pin actions" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.amd-runners) }} + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + if: needs.build-info.outputs.platform == 'linux/amd64' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -180,7 +201,7 @@ jobs: with: # octopin needs python 3.11 python-version: "3.11" - platform: "linux/amd64" + platform: ${{ needs.build-info.outputs.platform }} save-cache: true - name: "Run pin-versions" run: > @@ -194,7 +215,7 @@ jobs: needs: [build-info] uses: ./.github/workflows/basic-tests.yml with: - runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.runner-type }} run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} run-www-tests: ${{needs.build-info.outputs.run-www-tests}} run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}} @@ -204,7 +225,7 @@ jobs: canary-run: ${{needs.build-info.outputs.canary-run}} latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} use-uv: ${{needs.build-info.outputs.use-uv}} - platform: "linux/amd64" + platform: ${{ needs.build-info.outputs.platform }} shared-distributions-as-json: ${{needs.build-info.outputs.shared-distributions-as-json}} build-ci-images: @@ -217,8 +238,8 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} push-image: "false" upload-image-artifact: "true" upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }} @@ -240,8 +261,8 @@ jobs: packages: write id-token: write with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} @@ -262,8 +283,8 @@ jobs: uses: ./.github/workflows/generate-constraints.yml if: needs.build-info.outputs.ci-image-build == 'true' with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} python-versions: ${{ needs.build-info.outputs.python-versions }} generate-pypi-constraints: "true" @@ -281,8 +302,8 @@ jobs: id-token: write contents: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} run-mypy: ${{ needs.build-info.outputs.run-mypy }} mypy-checks: ${{ needs.build-info.outputs.mypy-checks }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -320,8 +341,8 @@ jobs: needs.build-info.outputs.latest-versions-only != 'true' && needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} canary-run: ${{ needs.build-info.outputs.canary-run }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -343,8 +364,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -361,8 +382,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" test-scope: "DB" @@ -390,8 +411,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" test-scope: "DB" @@ -419,8 +440,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" test-scope: "DB" @@ -438,7 +459,7 @@ jobs: skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} use-uv: ${{ needs.build-info.outputs.use-uv }} default-branch: ${{ needs.build-info.outputs.default-branch }} - if: needs.build-info.outputs.run-unit-tests == 'true' + if: needs.build-info.outputs.run-unit-tests == 'true' && needs.build-info.outputs.platform == 'linux/amd64' tests-mysql-providers: name: "MySQL tests: providers" @@ -448,8 +469,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" test-scope: "DB" @@ -467,7 +488,7 @@ jobs: skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} use-uv: ${{ needs.build-info.outputs.use-uv }} default-branch: ${{ needs.build-info.outputs.default-branch }} - if: needs.build-info.outputs.run-unit-tests == 'true' + if: needs.build-info.outputs.run-unit-tests == 'true' && needs.build-info.outputs.platform == 'linux/amd64' tests-sqlite-core: @@ -478,8 +499,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" test-name-separator: "" @@ -509,8 +530,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" test-name-separator: "" @@ -541,8 +562,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "" test-name-separator: "" @@ -571,8 +592,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "" test-name-separator: "" @@ -607,8 +628,8 @@ jobs: needs.build-info.outputs.full-tests-needed == 'true') with: default-branch: ${{ needs.build-info.outputs.default-branch }} - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} core-test-types-list-as-strings-in-json: > ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} providers-test-types-list-as-strings-in-json: > @@ -633,8 +654,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }} run-system-tests: ${{ needs.build-info.outputs.run-system-tests }} @@ -657,8 +678,8 @@ jobs: if: > needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" force-lowest-dependencies: "true" test-scope: "All" @@ -687,8 +708,8 @@ jobs: packages: read if: needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" force-lowest-dependencies: "true" test-scope: "All" @@ -718,8 +739,8 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} build-type: "Regular" push-image: "false" upload-image-artifact: "true" @@ -739,8 +760,8 @@ jobs: needs: [build-info, build-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} default-branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -759,8 +780,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -778,8 +799,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -794,7 +815,7 @@ jobs: tests-go-sdk: name: "Go SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.amd-runners) }} + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} timeout-minutes: 15 permissions: contents: read @@ -836,8 +857,8 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -883,8 +904,8 @@ jobs: - tests-with-lowest-direct-resolution-providers uses: ./.github/workflows/finalize-tests.yml with: - runners: ${{ needs.build-info.outputs.amd-runners }} - platform: "linux/amd64" + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} python-versions: ${{ needs.build-info.outputs.python-versions }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} branch: ${{ needs.build-info.outputs.default-branch }} @@ -901,6 +922,7 @@ jobs: notify-slack-failure: name: "Notify Slack on Failure" needs: + - build-info - finalize-tests if: github.event_name == 'schedule' && failure() && github.run_attempt == 1 runs-on: ["ubuntu-22.04"] @@ -914,12 +936,12 @@ jobs: # yamllint disable rule:line-length payload: | channel: "internal-airflow-ci-cd" - text: "🚨🕒 Failure Alert: Scheduled CI (AMD) on branch *${{ github.ref_name }}* 🕒🚨\n\n*Details:* " + text: "🚨🕒 Failure Alert: Scheduled CI (${{ needs.build-info.outputs.platform }}) on branch *${{ github.ref_name }}* 🕒🚨\n\n*Details:* " blocks: - type: "section" text: type: "mrkdwn" - text: "🚨🕒 Failure Alert: Scheduled CI (AMD) 🕒🚨\n\n*Details:* " + text: "🚨🕒 Failure Alert: Scheduled CI (${{ needs.build-info.outputs.platform }}) 🕒🚨\n\n*Details:* " # yamllint enable rule:line-length summarize-warnings: @@ -940,7 +962,7 @@ jobs: - tests-special - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers - runs-on: ${{ fromJSON(needs.build-info.outputs.amd-runners) }} + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} if: needs.build-info.outputs.run-unit-tests == 'true' steps: - name: "Cleanup repo" @@ -970,7 +992,7 @@ jobs: - name: "Upload artifact for summarized warnings" uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: test-summarized-amd-runner-warnings + name: "test-summarized-warnings" path: ./files/warn-summary-*.txt retention-days: 7 if-no-files-found: ignore diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml deleted file mode 100644 index a54504891416f..0000000000000 --- a/.github/workflows/ci-arm.yml +++ /dev/null @@ -1,860 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: Tests ARM -on: # yamllint disable-line rule:truthy - schedule: - - cron: '28 3,9,15,21 * * *' - push: - branches: - - v[0-9]+-[0-9]+-test - - providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+ - workflow_dispatch: -permissions: - # All other permissions are set to none by default - contents: read -env: - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_USERNAME: ${{ github.actor }} - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - VERBOSE: "true" - -concurrency: - group: ci-arm-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - - build-info: - name: "Build info" - # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners - runs-on: ["ubuntu-22.04"] - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - outputs: - all-python-versions-list-as-string: >- - ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} - amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} - arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} - basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} - canary-run: ${{ steps.source-run-info.outputs.canary-run }} - ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} - core-test-types-list-as-strings-in-json: >- - ${{ steps.selective-checks.outputs.core-test-types-list-as-strings-in-json }} - debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} - default-branch: ${{ steps.selective-checks.outputs.default-branch }} - default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} - default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} - default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} - default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} - default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} - default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} - default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} - disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} - docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} - docs-build: ${{ steps.selective-checks.outputs.docs-build }} - docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }} - excluded-providers-as-string: ${{ steps.selective-checks.outputs.excluded-providers-as-string }} - force-pip: ${{ steps.selective-checks.outputs.force-pip }} - full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} - has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} - helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} - include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} - individual-providers-test-types-list-as-strings-in-json: >- - ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }} - kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} - kubernetes-combos-list-as-string: >- - ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} - kubernetes-versions-list-as-string: >- - ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} - latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} - mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }} - mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} - mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} - postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} - postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} - prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} - # yamllint disable rule:line-length - providers-compatibility-tests-matrix: > - ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} - providers-test-types-list-as-strings-in-json: >- - ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} - pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} - python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} - python-versions: ${{ steps.selective-checks.outputs.python-versions }} - run-airflow-ctl-tests: ${{ steps.selective-checks.outputs.run-airflow-ctl-tests }} - run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} - run-api-codegen: ${{ steps.selective-checks.outputs.run-api-codegen }} - run-api-tests: ${{ steps.selective-checks.outputs.run-api-tests }} - run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} - run-go-sdk-tests: ${{ steps.selective-checks.outputs.run-go-sdk-tests }} - run-helm-tests: ${{ steps.selective-checks.outputs.run-helm-tests }} - run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} - run-mypy: ${{ steps.selective-checks.outputs.run-mypy }} - run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} - run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} - run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} - run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }} - run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} - selected-providers-list-as-string: >- - ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} - shared-distributions-as-json: ${{ steps.selective-checks.outputs.shared-distributions-as-json }} - skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }} - skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} - source-head-repo: ${{ steps.source-run-info.outputs.head-repo }} - source-head-ref: ${{ steps.source-run-info.outputs.head-ref }} - sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} - testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }} - testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }} - use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} - upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: Fetch incoming commit ${{ github.sha }} with its parent - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - - name: "Install Breeze" - uses: ./.github/actions/breeze - id: breeze - - name: "Get information about the Workflow" - id: source-run-info - run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} - env: - SKIP_BREEZE_SELF_UPGRADE_CHECK: "true" - - name: Selective checks - id: selective-checks - env: - PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" - COMMIT_REF: "${{ github.sha }}" - VERBOSE: "false" - run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} - - name: env - run: printenv - env: - PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} - GITHUB_CONTEXT: ${{ toJson(github) }} - - basic-tests: - name: "Basic tests" - needs: [build-info] - uses: ./.github/workflows/basic-tests.yml - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} - run-www-tests: ${{needs.build-info.outputs.run-www-tests}} - run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}} - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} - skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} - canary-run: ${{needs.build-info.outputs.canary-run}} - latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} - use-uv: ${{needs.build-info.outputs.use-uv}} - platform: "linux/arm64" - shared-distributions-as-json: ${{needs.build-info.outputs.shared-distributions-as-json}} - - build-ci-images: - name: Build CI images - needs: [build-info] - uses: ./.github/workflows/ci-image-build.yml - permissions: - contents: read - # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs - # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. - packages: write - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - push-image: "false" - upload-image-artifact: "true" - upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - branch: ${{ needs.build-info.outputs.default-branch }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - docker-cache: ${{ needs.build-info.outputs.docker-cache }} - disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} - if: needs.build-info.outputs.ci-image-build == 'true' - - additional-ci-image-checks: - name: "Additional CI image checks" - needs: [build-info, build-ci-images] - uses: ./.github/workflows/additional-ci-image-checks.yml - permissions: - contents: read - packages: write - id-token: write - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - python-versions: ${{ needs.build-info.outputs.python-versions }} - branch: ${{ needs.build-info.outputs.default-branch }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} - docker-cache: ${{ needs.build-info.outputs.docker-cache }} - disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} - canary-run: ${{ needs.build-info.outputs.canary-run }} - latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - - generate-constraints: - name: "Generate constraints" - needs: [build-info, build-ci-images] - uses: ./.github/workflows/generate-constraints.yml - if: needs.build-info.outputs.ci-image-build == 'true' - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - generate-pypi-constraints: "true" - # generate no providers constraints only in canary builds - they take quite some time to generate - # they are not needed for regular builds, they are only needed to update constraints in canaries - generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - - ci-image-checks: - name: "CI image checks" - needs: [build-info, build-ci-images] - uses: ./.github/workflows/ci-image-checks.yml - permissions: - id-token: write - contents: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - run-mypy: ${{ needs.build-info.outputs.run-mypy }} - mypy-checks: ${{ needs.build-info.outputs.mypy-checks }} - python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} - branch: ${{ needs.build-info.outputs.default-branch }} - canary-run: ${{ needs.build-info.outputs.canary-run }} - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }} - latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} - basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} - ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - docs-build: ${{ needs.build-info.outputs.docs-build }} - run-api-codegen: ${{ needs.build-info.outputs.run-api-codegen }} - default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - source-head-repo: ${{ needs.build-info.outputs.source-head-repo }} - source-head-ref: ${{ needs.build-info.outputs.source-head-ref }} - secrets: - DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} - DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} - - providers: - name: "provider distributions tests" - uses: ./.github/workflows/test-providers.yml - needs: [build-info, build-ci-images] - permissions: - contents: read - packages: read - if: > - needs.build-info.outputs.skip-providers-tests != 'true' && - needs.build-info.outputs.latest-versions-only != 'true' && - needs.build-info.outputs.run-unit-tests == 'true' - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - canary-run: ${{ needs.build-info.outputs.canary-run }} - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }} - # yamllint disable rule:line-length - providers-compatibility-tests-matrix: > - ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - providers-test-types-list-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - - tests-helm: - name: "Helm tests" - uses: ./.github/workflows/helm-tests.yml - needs: [build-info, build-ci-images] - permissions: - contents: read - packages: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - use-uv: ${{ needs.build-info.outputs.use-uv }} - if: > - needs.build-info.outputs.run-helm-tests == 'true' && - needs.build-info.outputs.default-branch == 'main' && - needs.build-info.outputs.latest-versions-only != 'true' - - tests-postgres-core: - name: "Postgres tests: core" - uses: ./.github/workflows/run-unit-tests.yml - needs: [build-info, build-ci-images] - permissions: - contents: read - packages: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - backend: "postgres" - test-name: "Postgres" - test-scope: "DB" - test-group: "core" - python-versions: ${{ needs.build-info.outputs.python-versions }} - backend-versions: ${{ needs.build-info.outputs.postgres-versions }} - excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} - excludes: ${{ needs.build-info.outputs.postgres-exclude }} - test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-migration-tests: "true" - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - if: needs.build-info.outputs.run-unit-tests == 'true' - - tests-postgres-providers: - name: "Postgres tests: providers" - uses: ./.github/workflows/run-unit-tests.yml - needs: [build-info, build-ci-images] - permissions: - contents: read - packages: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - backend: "postgres" - test-name: "Postgres" - test-scope: "DB" - test-group: "providers" - python-versions: ${{ needs.build-info.outputs.python-versions }} - backend-versions: ${{ needs.build-info.outputs.postgres-versions }} - excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} - excludes: ${{ needs.build-info.outputs.postgres-exclude }} - test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-migration-tests: "true" - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - if: needs.build-info.outputs.run-unit-tests == 'true' - - tests-sqlite-core: - name: "Sqlite tests: core" - uses: ./.github/workflows/run-unit-tests.yml - needs: [build-info, build-ci-images] - permissions: - contents: read - packages: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - backend: "sqlite" - test-name: "Sqlite" - test-name-separator: "" - test-scope: "DB" - test-group: "core" - python-versions: ${{ needs.build-info.outputs.python-versions }} - # No versions for sqlite - backend-versions: "['']" - excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} - excludes: ${{ needs.build-info.outputs.sqlite-exclude }} - test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - run-migration-tests: "true" - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - if: needs.build-info.outputs.run-unit-tests == 'true' - - tests-sqlite-providers: - name: "Sqlite tests: providers" - uses: ./.github/workflows/run-unit-tests.yml - needs: [build-info, build-ci-images] - permissions: - contents: read - packages: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - backend: "sqlite" - test-name: "Sqlite" - test-name-separator: "" - test-scope: "DB" - test-group: "providers" - python-versions: ${{ needs.build-info.outputs.python-versions }} - # No versions for sqlite - backend-versions: "['']" - excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} - excludes: ${{ needs.build-info.outputs.sqlite-exclude }} - test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - run-migration-tests: "true" - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - if: needs.build-info.outputs.run-unit-tests == 'true' - - tests-non-db-core: - name: "Non-DB tests: core" - uses: ./.github/workflows/run-unit-tests.yml - needs: [build-info, build-ci-images] - permissions: - contents: read - packages: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - backend: "sqlite" - test-name: "" - test-name-separator: "" - test-scope: "Non-DB" - test-group: "core" - python-versions: ${{ needs.build-info.outputs.python-versions }} - # No versions for non-db - backend-versions: "['']" - excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} - excludes: ${{ needs.build-info.outputs.sqlite-exclude }} - test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - if: needs.build-info.outputs.run-unit-tests == 'true' - - tests-non-db-providers: - name: "Non-DB tests: providers" - uses: ./.github/workflows/run-unit-tests.yml - needs: [build-info, build-ci-images] - permissions: - contents: read - packages: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - backend: "sqlite" - test-name: "" - test-name-separator: "" - test-scope: "Non-DB" - test-group: "providers" - python-versions: ${{ needs.build-info.outputs.python-versions }} - # No versions for non-db - backend-versions: "['']" - excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} - excludes: ${{ needs.build-info.outputs.sqlite-exclude }} - test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - if: needs.build-info.outputs.run-unit-tests == 'true' - - tests-special: - name: "Special tests" - uses: ./.github/workflows/special-tests.yml - needs: [build-info, build-ci-images] - permissions: - contents: read - packages: read - if: > - needs.build-info.outputs.run-unit-tests == 'true' && - (needs.build-info.outputs.canary-run == 'true' || - needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' || - needs.build-info.outputs.full-tests-needed == 'true') - with: - default-branch: ${{ needs.build-info.outputs.default-branch }} - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - core-test-types-list-as-strings-in-json: > - ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} - providers-test-types-list-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - python-versions: ${{ needs.build-info.outputs.python-versions }} - default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} - excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} - canary-run: ${{ needs.build-info.outputs.canary-run }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - - tests-with-lowest-direct-resolution-core: - name: "Low dep tests:core" - needs: [build-info, build-ci-images] - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - if: > - needs.build-info.outputs.run-unit-tests == 'true' - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - test-name: "LowestDeps" - force-lowest-dependencies: "true" - test-scope: "All" - test-group: "core" - backend: "sqlite" - python-versions: ${{ needs.build-info.outputs.python-versions }} - backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']" - excluded-providers-as-string: "" - excludes: "[]" - test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - monitor-delay-time-in-seconds: 120 - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - - tests-with-lowest-direct-resolution-providers: - name: "Low dep tests: providers" - needs: [build-info, build-ci-images] - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - if: needs.build-info.outputs.run-unit-tests == 'true' - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - test-name: "LowestDeps" - force-lowest-dependencies: "true" - test-scope: "All" - test-group: "providers" - backend: "sqlite" - python-versions: ${{ needs.build-info.outputs.python-versions }} - backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']" - excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} - excludes: "[]" - test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.individual-providers-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - run-coverage: ${{ needs.build-info.outputs.run-coverage }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - monitor-delay-time-in-seconds: 120 - skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - - build-prod-images: - name: Build PROD images - needs: [build-info, build-ci-images, generate-constraints] - uses: ./.github/workflows/prod-image-build.yml - permissions: - contents: read - # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs - # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. - packages: write - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - build-type: "Regular" - push-image: "false" - upload-image-artifact: "true" - upload-package-artifact: "true" - python-versions: ${{ needs.build-info.outputs.python-versions }} - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - branch: ${{ needs.build-info.outputs.default-branch }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - docker-cache: ${{ needs.build-info.outputs.docker-cache }} - disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} - prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} - - additional-prod-image-tests: - name: "Additional PROD image tests" - needs: [build-info, build-prod-images, generate-constraints] - uses: ./.github/workflows/additional-prod-image-tests.yml - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - default-branch: ${{ needs.build-info.outputs.default-branch }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - docker-cache: ${{ needs.build-info.outputs.docker-cache }} - disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - canary-run: ${{ needs.build-info.outputs.canary-run }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - if: needs.build-info.outputs.prod-image-build == 'true' - - tests-kubernetes: - name: "Kubernetes tests" - uses: ./.github/workflows/k8s-tests.yml - needs: [build-info, build-prod-images] - permissions: - contents: read - packages: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }} - if: > - ( needs.build-info.outputs.run-kubernetes-tests == 'true' || - needs.build-info.outputs.run-helm-tests == 'true') - - tests-task-sdk: - name: "Task SDK tests" - uses: ./.github/workflows/airflow-distributions-tests.yml - needs: [build-info, build-ci-images] - permissions: - contents: read - packages: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - python-versions: ${{ needs.build-info.outputs.python-versions }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - canary-run: ${{ needs.build-info.outputs.canary-run }} - distribution-name: "task-sdk" - distribution-cmd-format: "prepare-task-sdk-distributions" - test-type: "task-sdk-tests" - use-local-venv: 'false' - test-timeout: 20 - if: needs.build-info.outputs.run-task-sdk-tests == 'true' - - tests-go-sdk: - name: "Go SDK tests" - needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.arm-runners) }} - timeout-minutes: 15 - permissions: - contents: read - packages: read - if: needs.build-info.outputs.run-go-sdk-tests == 'true' - env: - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_USERNAME: ${{ github.actor }} - VERBOSE: "true" - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - # keep this in sync with go.mod in go-sdk/ - - name: Setup Go - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version: 1.24 - cache-dependency-path: go-sdk/go.sum - # keep this in sync with go.mod in go-sdk/ - - name: Setup Gotestsum - shell: bash - run: | - go install gotest.tools/gotestsum@c4a0df2e75a225d979a444342dd3db752b53619f # v1.13.0 - gotestsum --version - - name: "Cleanup dist files" - run: rm -fv ./dist/* - - name: Run Go tests - working-directory: ./go-sdk - run: gotestsum --format github-actions ./... - - tests-airflow-ctl: - name: "Airflow CTL tests" - uses: ./.github/workflows/airflow-distributions-tests.yml - needs: [build-info] - permissions: - contents: read - packages: read - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - python-versions: ${{ needs.build-info.outputs.python-versions }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - canary-run: ${{ needs.build-info.outputs.canary-run }} - distribution-name: "airflow-ctl" - distribution-cmd-format: "prepare-airflow-ctl-distributions" - test-type: "airflow-ctl-tests" - use-local-venv: 'true' - test-timeout: 20 - if: needs.build-info.outputs.run-airflow-ctl-tests == 'true' - - finalize-tests: - name: Finalize tests - permissions: - contents: write - packages: write - # This will fire when all the jobs from "needs" are either successful or skipped - if: always() && !failure() && !cancelled() - needs: - - additional-ci-image-checks - - additional-prod-image-tests - - basic-tests - - build-info - - build-prod-images - - ci-image-checks - - generate-constraints - - providers - - tests-helm - - tests-kubernetes - - tests-non-db-core - - tests-non-db-providers - - tests-postgres-core - - tests-postgres-providers - - tests-sqlite-core - - tests-sqlite-providers - - tests-task-sdk - - tests-airflow-ctl - - tests-go-sdk - - tests-with-lowest-direct-resolution-core - - tests-with-lowest-direct-resolution-providers - uses: ./.github/workflows/finalize-tests.yml - with: - runners: ${{ needs.build-info.outputs.arm-runners }} - platform: "linux/arm64" - python-versions: ${{ needs.build-info.outputs.python-versions }} - python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} - branch: ${{ needs.build-info.outputs.default-branch }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - default-python-version: "${{ needs.build-info.outputs.default-python-version }}" - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - docker-cache: ${{ needs.build-info.outputs.docker-cache }} - disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} - canary-run: ${{ needs.build-info.outputs.canary-run }} - use-uv: ${{ needs.build-info.outputs.use-uv }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - - notify-slack-failure: - name: "Notify Slack on Failure" - needs: - - finalize-tests - if: github.event_name == 'schedule' && failure() && github.run_attempt == 1 - runs-on: ["ubuntu-22.04"] - steps: - - name: Notify Slack - id: slack - uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 - with: - method: chat.postMessage - token: ${{ env.SLACK_BOT_TOKEN }} - # yamllint disable rule:line-length - payload: | - channel: "internal-airflow-ci-cd" - text: "🚨🕒 Failure Alert: Scheduled CI (ARM) on branch *${{ github.ref_name }}* 🕒🚨\n\n*Details:* " - blocks: - - type: "section" - text: - type: "mrkdwn" - text: "🚨🕒 Failure Alert: Scheduled CI (ARM) 🕒🚨\n\n*Details:* " - # yamllint enable rule:line-length - - summarize-warnings: - timeout-minutes: 15 - name: "Summarize warnings" - needs: - - build-info - - tests-non-db-core - - tests-non-db-providers - - tests-postgres-core - - tests-postgres-providers - - tests-sqlite-core - - tests-sqlite-providers - - tests-task-sdk - - tests-airflow-ctl - - tests-special - - tests-with-lowest-direct-resolution-core - - tests-with-lowest-direct-resolution-providers - runs-on: ${{ fromJSON(needs.build-info.outputs.arm-runners) }} - if: needs.build-info.outputs.run-unit-tests == 'true' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: "Free up disk space" - shell: bash - run: ./scripts/tools/free_up_disk_space.sh - - name: "Download all test warning artifacts from the current build" - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - path: ./artifacts - pattern: test-warnings-* - - name: "Setup python" - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "${{ inputs.default-python-version }}" - - name: "Summarize all warnings" - run: | - ./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \ - --pattern "**/warnings-*.txt" \ - --output ./files - - name: "Upload artifact for summarized warnings" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: test-summarized-arm-runner-warnings - path: ./files/warn-summary-*.txt - retention-days: 7 - if-no-files-found: ignore - overwrite: true diff --git a/.github/workflows/ci-notification.yml b/.github/workflows/ci-notification.yml index 60c2476c2dd81..d44c2d2706d7a 100644 --- a/.github/workflows/ci-notification.yml +++ b/.github/workflows/ci-notification.yml @@ -36,8 +36,8 @@ jobs: workflow-status: strategy: matrix: - branch: ["v3-0-test"] - workflow-id: ["ci-amd.yml", "ci-arm.yml"] + branch: ["v3-1-test"] + workflow-id: ["ci-amd-arm.yml"] runs-on: ubuntu-latest steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index f2541d0626caa..9ca30c924e865 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -189,6 +189,13 @@ jobs: ref: ${{ inputs.ref }} fetch-depth: 0 fetch-tags: true + - name: "Free up disk space" + shell: bash + run: ./scripts/tools/free_up_disk_space.sh + - name: "Make /mnt writeable" + run: ./scripts/ci/make_mnt_writeable.sh + - name: "Move docker to /mnt" + run: ./scripts/ci/move_docker_to_mnt.sh - name: "Apply patch commits if provided" run: | if [[ "${APPLY_COMMITS}" != "" ]]; then @@ -305,7 +312,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} INCLUDE_SUCCESS_OUTPUTS: false - PYTHON_MAJOR_MINOR_VERSION: 3.10 + PYTHON_MAJOR_MINOR_VERSION: "3.10" VERBOSE: "true" steps: - name: "Cleanup repo" diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 8fc99b537d5eb..6c1ebaef4e0d1 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -58,7 +58,7 @@ jobs: AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} AMD_ONLY: ${{ github.event.inputs.amdOnly }} LIMIT_PYTHON_VERSIONS: ${{ github.event.inputs.limitPythonVersions }} - UV_VERSION: "0.9.4" # Keep this comment to allow automatic replacement of uv version + UV_VERSION: "0.9.7" # Keep this comment to allow automatic replacement of uv version if: contains(fromJSON('[ "ashb", "eladkal", diff --git a/.gitignore b/.gitignore index d3f1810a46e48..1528de626bdac 100644 --- a/.gitignore +++ b/.gitignore @@ -171,8 +171,6 @@ npm-debug.log* yarn-debug.log* yarn-error.log* pnpm-debug.log* -.vscode/* -!.vscode/extensions.json /.vite/ # Exclude the ui .vite dir airflow-core/src/airflow/ui/.vite/ @@ -275,3 +273,9 @@ _api/ #while running go tests inside the go-sdk, it can generate log files for dags, ignore all logs go-sdk/**/*.log + +# E2e tests +_e2e_test_report.json + +# UV cache +.uv-cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ec142eaf0f70..8d04ad98097cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -123,20 +123,12 @@ repos: - --license-filepath - scripts/ci/license-templates/LICENSE.txt - --fuzzy-match-generates-todo - - id: insert-license - name: Add license for all Helm template files - files: ^chart/templates/.* - args: - - --comment-style - - "{{/*||*/}}" - - --license-filepath - - scripts/ci/license-templates/LICENSE.txt - - --fuzzy-match-generates-todo - id: insert-license name: Add license for all YAML files except Helm templates exclude: > (?x) - ^\.github/.*$|^chart/templates/.*| + ^\.github/.*$| + ^chart/templates/.*| .*reproducible_build\.yaml$| ^.*/v2.*\.yaml$| ^.*/openapi/_private_ui.*\.yaml$| @@ -197,14 +189,6 @@ repos: language: python pass_filenames: false require_serial: true - - id: update-chart-dependencies - name: Update chart dependencies to latest (manual) - entry: ./scripts/ci/prek/update_chart_dependencies.py - stages: ['manual'] - language: python - files: ^\.pre-commit-config\.yaml$|^scripts/ci/prek/update_build_dependencies\.py$ - pass_filenames: false - require_serial: true - id: check-taskinstance-tis-attrs name: Check that TI and TIS have the same attributes entry: ./scripts/ci/prek/check_ti_vs_tis_attributes.py @@ -212,13 +196,6 @@ repos: files: ^airflow-core/src/airflow/models/taskinstance\.py$|^airflow-core/src/airflow/models/taskinstancehistory\.py$ pass_filenames: false require_serial: true - - id: check-deferrable-default - name: Check and fix default value of default_deferrable - language: python - entry: ./scripts/ci/prek/check_deferrable_default.py - pass_filenames: false - # libcst doesn't have source wheels for all PY except PY3.12, excluding it - files: ^(providers/.*/)?airflow/.*/(sensors|operators)/.*\.py$ - repo: https://github.com/adamchainz/blacken-docs rev: dda8db18cfc68df532abf33b185ecd12d5b7b326 # frozen: 1.20.0 hooks: @@ -232,7 +209,7 @@ repos: - --target-version=py313 alias: blacken-docs additional_dependencies: - - 'black==25.1.0' + - 'black==25.9.0' - repo: https://github.com/pre-commit/pre-commit-hooks rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 hooks: @@ -271,15 +248,6 @@ repos: ^dev/breeze/doc/images/output.*$| ^.*/openapi-gen/.*$| ^airflow-ctl/docs/images/.*\.svg$ - - id: pretty-format-json - name: Format JSON files - args: - - --autofix - - --no-sort-keys - - --indent - - "4" - files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$ - pass_filenames: true - repo: https://github.com/pre-commit/pygrep-hooks rev: 3a6eb0fadf60b3cccfd80bad9dbb6fae7e47b316 # frozen: v1.10.0 hooks: @@ -342,7 +310,7 @@ repos: - --skip=providers/.*/src/airflow/providers/*/*.rst,providers/*/docs/changelog.rst,docs/*/commits.rst,providers/*/docs/commits.rst,providers/*/*/docs/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md,*.svg - --exclude-file=.codespellignorelines - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: 122d24ec728f140b38330ae8b04e3c5fe8b774c5 # frozen: v1.15.2 + rev: 77c6d22b0c515f1635e07a99140d2230db2e2c97 # frozen: v1.16.1 hooks: - id: zizmor name: Run zizmor to check for github workflow syntax errors @@ -359,12 +327,6 @@ repos: # changes quickly - especially when we want the early modifications from the first local group # to be applied before the non-local prek hooks are run hooks: - - id: update-providers-dependencies - name: Update dependencies for providers - entry: ./scripts/ci/prek/update_providers_dependencies.py - language: python - always_run: true - pass_filenames: false - id: check-shared-distributions-structure name: Check shared distributions structure entry: ./scripts/ci/prek/check_shared_distributions_structure.py @@ -377,34 +339,6 @@ repos: language: python pass_filenames: false files: ^shared/.*$|^.*/pyproject.toml$|^.*/_shared/.*$ - - id: validate-operators-init - name: No templated field logic checks in operator __init__ - description: Prevent templated field logic checks in operators' __init__ - language: python - entry: ./scripts/ci/prek/validate_operators_init.py - pass_filenames: true - files: ^providers/.*/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$ - - id: update-providers-build-files - name: Update providers build files - entry: ./scripts/ci/prek/update_providers_build_files.py - language: python - pass_filenames: true - files: | - (?x) - ^providers/[^\/]*/src/airflow/providers/[^\/]*/__init__\.py$| - ^providers/[^\/]*/[^\/]*/src/airflow/providers/[^\/]*/[^\/]*/__init__\.py$| - ^providers/.*/pyproject\.toml$| - ^providers/.*/provider\.yaml$| - ^airflow_breeze/templates/PROVIDER__INIT__PY_TEMPLATE\.py\.jinja2$| - ^airflow_breeze/templates/get_provider_info_TEMPLATE\.py\.jinja2$| - ^airflow_breeze/templates/PROVIDER_README_TEMPLATE\.rst\.jinja2$ - require_serial: true - - id: check-airflow-v-imports-in-tests - name: Check AIRFLOW_V imports in tests - language: python - entry: ./scripts/ci/prek/check_airflow_v_imports_in_tests.py - pass_filenames: true - files: ^providers/.*/tests/.+\.py$ - id: ruff name: Run 'ruff' for extremely fast Python linting description: "Run 'ruff' for extremely fast Python linting" @@ -413,7 +347,7 @@ repos: types_or: [python, pyi] args: [--fix] require_serial: true - additional_dependencies: ['ruff==0.14.1'] + additional_dependencies: ['ruff==0.14.3'] exclude: ^airflow-core/tests/unit/dags/test_imports\.py$|^performance/tests/test_.*\.py$ - id: ruff-format name: Run 'ruff format' @@ -494,12 +428,6 @@ repos: language: python files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$ pass_filenames: false - - id: check-sql-dependency-common-data-structure - name: Check dependency of SQL providers - description: Check dependency of SQL Providers with common data structure - entry: ./scripts/ci/prek/check_common_sql_dependency.py - language: python - files: ^providers/.*/src/airflow/providers/.*/hooks/.*\.py$ - id: check-extra-packages-references name: Checks setup extra packages description: Checks if all the extras defined in hatch_build.py are listed in extra-packages-ref.rst file @@ -520,15 +448,9 @@ repos: files: > (?x) ^airflow-core/docs/.*/diagram_[^/]*\.py$| - ^docs/images/.*\.py$ + ^docs/images/.*\.py$| + ^airflow-ctl/docs/images/diagrams/.*\.py$ pass_filenames: true - - id: generate-volumes-for-sources - name: Generate volumes for docker compose - entry: ./scripts/ci/prek/generate_volumes_for_sources.py - language: python - files: ^providers/.*/provider\.yaml$ - pass_filenames: false - require_serial: true - id: prevent-deprecated-sqlalchemy-usage name: Prevent deprecated sqlalchemy usage entry: ./scripts/ci/prek/prevent_deprecated_sqlalchemy_usage.py @@ -537,7 +459,6 @@ repos: (?x) ^airflow-ctl.*\.py$| ^airflow-core/src/airflow/models/.*\.py$| - ^providers/fab/.*\.py$| ^task_sdk.*\.py$ pass_filenames: true - id: update-supported-versions @@ -554,10 +475,9 @@ repos: files: > (?x) ^scripts/ci/prek/version_heads_map\.py$| - ^airflow-core/src/airflow/migrations/versions/.*$|^airflow-core/src/airflow/migrations/versions| - ^providers/fab/src/airflow/providers/fab/migrations/versions/.*$|^providers/fab/src/airflow/providers/fab/migrations/versions| - ^airflow-core/src/airflow/utils/db\.py$| - ^providers/fab/src/airflow/providers/fab/auth_manager/models/db\.py$ + ^airflow-core/src/airflow/migrations/versions/.*$| + ^airflow-core/src/airflow/migrations/versions| + ^airflow-core/src/airflow/utils/db\.py$ - id: update-version name: Update versions in docs entry: ./scripts/ci/prek/update_versions.py @@ -570,75 +490,6 @@ repos: entry: "pydevd.*settrace\\(" pass_filenames: true files: \.py$ - - id: check-pytest-mark-db-test-in-providers - language: pygrep - name: Check pytest.mark.db_test use in providers - entry: pytest\.mark\.db_test - pass_filenames: true - # Here we should add providers that are already free from the pytest.mark.db_test - # and we want to keep them clean and only use non-db-tests - files: > - (?x) - ^providers/airbyte/.*\.py$| - ^providers/apache/beam/.*\.py$| - ^providers/apache/flink/.*\.py$| - ^providers/apache/iceberg/.*\.py$| - ^providers/apache/kafka/.*\.py$| - ^providers/apprise/.*\.py$| - ^providers/arangodb/.*\.py$| - ^providers/asana/.*\.py$| - ^providers/atlassian/jira/.*\.py$| - ^providers/cloudant/.*\.py$| - ^providers/cohere/.*\.py$| - ^providers/common/compat/.*\.py$| - ^providers/common/messaging/.*\.py$| - ^providers/datadog/.*\.py$| - ^providers/dingding/.*\.py$| - ^providers/discord/.*\.py$| - ^providers/exasol/.*\.py$| - ^providers/facebook/.*\.py$| - ^providers/ftp/.*\.py$| - ^providers/grpc/.*\.py$| - ^providers/hashicorp/.*\.py$| - ^providers/imap/.*\.py$| - ^providers/influxdb/.*\.py$| - ^providers/jdbc/.*\.py$| - ^providers/jenkins/.*\.py$| - ^providers/mongo/.*\.py$| - ^providers/microsoft/psrp/.*\.py$| - ^providers/microsoft/winrm/.*\.py$| - ^providers/neo4j/.*\.py$| - ^providers/odbc/.*\.py$| - ^providers/openai/.*\.py$| - ^providers/openfaas/.*\.py$| - ^providers/opsgenie/.*\.py$| - ^providers/oracle/.*\.py$| - ^providers/pagerduty/.*\.py$| - ^providers/pgvector/.*\.py$| - ^providers/pinecone/.*\.py$| - ^providers/postgres/.*\.py$| - ^providers/presto/.*\.py$| - ^providers/segment/.*\.py$| - ^providers/sendgrid/.*\.py$| - ^providers/singularity/.*\.py$| - ^providers/slack/.*\.py$| - ^providers/smtp/.*\.py$| - ^providers/tableau/.*\.py$| - ^providers/teradata/.*\.py$| - ^providers/trino/.*\.py$| - ^providers/vertica/.*\.py$| - ^providers/yandex/.*\.py$| - ^providers/zendesk/.*\.py$ - - id: check-links-to-example-dags-do-not-use-hardcoded-versions - name: Verify no hard-coded version in example dags - description: The links to example dags should use |version| as version specification - language: pygrep - entry: > - (?i) - .*https://github.*/main/providers/.*/src/airflow/providers/.*/example_dags/| - .*https://github.*/master/providers/.*/src/airflow/providers/.*/example_dags/ - pass_filenames: true - files: ^providers/.*/docs/.*\.rst - id: check-safe-filter-usage-in-html language: pygrep name: Don't use safe in templates @@ -653,13 +504,6 @@ repos: entry: "^\\s*from airflow\\.providers.(?!standard.)" pass_filenames: true files: ^airflow-core/src/airflow/example_dags/.*\.py$ - - id: check-no-airflow-deprecation-in-providers - language: pygrep - name: Do not use DeprecationWarning in providers - description: Use AirflowProviderDeprecationWarning in providers - entry: "^\\s*DeprecationWarning*" - pass_filenames: true - files: ^providers/.*/src/airflow/providers/.*\.py$ - id: check-urlparse-usage-in-code language: pygrep name: Don't use urlparse in code @@ -692,91 +536,87 @@ repos: pass_filenames: true exclude: > (?x) - ^airflow-core/src/airflow/ui/src/i18n/config\.ts$| - ^airflow-core/src/airflow/ui/openapi-gen/| - ^airflow-core/src/airflow/ui/public/i18n/locales/de/README\.md$| + ^airflow-core/docs/.*commits\.rst$| + ^airflow-core/newsfragments/41368\.significant\.rst$| + ^airflow-core/newsfragments/41761.significant\.rst$| + ^airflow-core/newsfragments/43349\.significant\.rst$| + ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/pnpm-lock\.yaml$| ^airflow-core/src/airflow/cli/commands/local_commands/fastapi_api_command\.py$| ^airflow-core/src/airflow/config_templates/| ^airflow-core/src/airflow/models/baseoperator\.py$| ^airflow-core/src/airflow/operators/__init__\.py$| - ^providers/common/sql/tests/provider_tests/common/sql/operators/test_sql_execute\.py$| + ^airflow-core/src/airflow/serialization/serialized_objects\.py$| + ^airflow-core/src/airflow/ui/openapi-gen/| + ^airflow-core/src/airflow/ui/pnpm-lock\.yaml$| + ^airflow-core/src/airflow/ui/public/i18n/locales/de/README\.md$| + ^airflow-core/src/airflow/ui/src/i18n/config\.ts$| + ^airflow-core/src/airflow/utils/db\.py$| + ^airflow-core/src/airflow/utils/trigger_rule\.py$| + ^airflow-core/tests/| + ^.*changelog\.(rst|txt)$| + ^.*CHANGELOG\.(rst|txt)$| + ^chart/values.schema\.json$| + ^.*commits\.(rst|txt)$| + ^.*/conf_constants\.py$| + ^.*/conf\.py$| + ^contributing-docs/03_contributors_quick_start\.rst$| + ^dev/| + ^devel-common/src/docs/README\.rst$| + ^devel-common/src/sphinx_exts/removemarktransform\.py| + ^devel-common/src/tests_common/test_utils/db\.py| + .*/dist/.*| + ^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store\.rst$| + git| + ^helm-tests/tests/chart_utils/helm_template_generator\.py$| + ^helm-tests/tests/chart_utils/ingress-networking-v1beta1\.json$| + package-lock\.json$| + ^.*\.(png|gif|jp[e]?g|svg|tgz|lock)$| + ^\.pre-commit-config\.yaml$| + ^.*/provider_conf\.py$| + ^providers/\.pre-commit-config\.yaml$| ^providers/amazon/src/airflow/providers/amazon/aws/hooks/emr\.py$| ^providers/amazon/src/airflow/providers/amazon/aws/operators/emr\.py$| ^providers/apache/cassandra/src/airflow/providers/apache/cassandra/hooks/cassandra\.py$| + ^providers/apache/hdfs/docs/connections\.rst$| ^providers/apache/hive/src/airflow/providers/apache/hive/operators/hive_stats\.py$| ^providers/apache/hive/src/airflow/providers/apache/hive/transfers/vertica_to_hive\.py$| + ^providers/apache/kafka/docs/connections/kafka\.rst$| + ^providers/apache/spark/docs/decorators/pyspark\.rst$| ^providers/apache/spark/src/airflow/providers/apache/spark/decorators/| ^providers/apache/spark/src/airflow/providers/apache/spark/hooks/| ^providers/apache/spark/src/airflow/providers/apache/spark/operators/| + ^providers/cncf/kubernetes/docs/operators\.rst$| + ^providers/common/sql/tests/provider_tests/common/sql/operators/test_sql_execute\.py$| + ^providers/edge3/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml$| ^providers/exasol/src/airflow/providers/exasol/hooks/exasol\.py$| + ^providers/fab/docs/auth-manager/webserver-authentication\.rst$| ^providers/fab/src/airflow/providers/fab/auth_manager/security_manager/| ^providers/fab/src/airflow/providers/fab/www/static/| ^providers/fab/src/airflow/providers/fab/www/templates/| + ^providers/google/docs/operators/cloud/kubernetes_engine\.rst$| ^providers/google/src/airflow/providers/google/cloud/hooks/bigquery\.py$| ^providers/google/src/airflow/providers/google/cloud/operators/cloud_build\.py$| ^providers/google/src/airflow/providers/google/cloud/operators/dataproc\.py$| ^providers/google/src/airflow/providers/google/cloud/operators/mlengine\.py$| ^providers/keycloak/src/airflow/providers/keycloak/auth_manager/cli/definition.py| + ^providers/microsoft/azure/docs/connections/azure_cosmos\.rst$| ^providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/cosmos\.py$| ^providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/hooks/winrm\.py$| - ^airflow-core/docs/.*commits\.rst$| ^providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/operators/winrm\.py$| ^providers/opsgenie/src/airflow/providers/opsgenie/hooks/opsgenie\.py$| ^providers/redis/src/airflow/providers/redis/provider\.yaml$| - ^airflow-core/src/airflow/serialization/serialized_objects\.py$| - ^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/pnpm-lock\.yaml$| - ^airflow-core/src/airflow/ui/pnpm-lock\.yaml$| - ^airflow-core/src/airflow/utils/db\.py$| - ^airflow-core/src/airflow/utils/trigger_rule\.py$| - ^chart/values.schema\.json$| - ^helm-tests/tests/chart_utils/helm_template_generator\.py$| - ^helm-tests/tests/chart_utils/ingress-networking-v1beta1\.json$| - ^dev/| - ^devel-common/src/docs/README\.rst$| - ^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store\.rst$| - ^providers/apache/kafka/docs/connections/kafka\.rst$| - ^providers/apache/hdfs/docs/connections\.rst$| - ^providers/apache/spark/docs/decorators/pyspark\.rst$| - ^providers/microsoft/azure/docs/connections/azure_cosmos\.rst$| - ^providers/fab/docs/auth-manager/webserver-authentication\.rst$| - ^providers/google/docs/operators/cloud/kubernetes_engine\.rst$| - ^providers/cncf/kubernetes/docs/operators\.rst$| - ^.*/conf\.py$| - ^.*/conf_constants\.py$| - ^.*/provider_conf\.py$| - ^devel-common/src/sphinx_exts/removemarktransform\.py| - ^devel-common/src/tests_common/test_utils/db\.py| - ^airflow-core/newsfragments/41761.significant\.rst$| - ^scripts/ci/prek/vendor_k8s_json_schema\.py$| - ^scripts/ci/docker-compose/integration-keycloak\.yml$| - ^scripts/ci/docker-compose/keycloak/keycloak-entrypoint\.sh$| - ^airflow-core/tests/| ^providers/.*/tests/| - ^\.pre-commit-config\.yaml$| - ^.*CHANGELOG\.(rst|txt)$| - ^.*changelog\.(rst|txt)$| - ^.*commits\.(rst|txt)$| + .rat-excludes| ^.*RELEASE_NOTES\.rst$| - ^contributing-docs/03_contributors_quick_start\.rst$| - ^.*\.(png|gif|jp[e]?g|svg|tgz|lock)$| - git| - ^airflow-core/newsfragments/43349\.significant\.rst$| - ^airflow-core/newsfragments/41368\.significant\.rst$| - .*/dist/.*| - package-lock\.json$| - ^providers/edge3/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml$ + ^scripts/ci/docker-compose/integration-keycloak\.yml$| + ^scripts/ci/docker-compose/keycloak/keycloak-entrypoint\.sh$| + ^scripts/ci/prek/vendor_k8s_json_schema\.py$ - id: check-base-operator-partial-arguments name: Check BaseOperator and partial() arguments language: python entry: ./scripts/ci/prek/check_base_operator_partial_arguments.py pass_filenames: false files: ^airflow-core/src/airflow/models/(?:base|mapped)operator\.py$ - - id: check-init-decorator-arguments - name: Sync model __init__ and decorator arguments - language: python - entry: ./scripts/ci/prek/check_init_decorator_arguments.py - pass_filenames: false - files: ^task-sdk/src/airflow/sdk/definitions/dag\.py$|^task-sdk/src/airflow/sdk/definitions/decorators/task_group\.py$ - id: check-template-context-variable-in-sync name: Sync template context variable refs language: python @@ -809,27 +649,6 @@ repos: ^airflow-core/src/airflow/operators/.*$| ^providers/.*/src/airflow/providers/.*$| ^providers/.*/src/airflow/providers/standard/sensors/.*$ - - id: check-base-operator-usage - language: pygrep - name: Check BaseOperator other imports - description: Make sure BaseOperator is imported from airflow.models outside of core - entry: "from airflow\\.models\\.baseoperator import.* BaseOperator" - pass_filenames: true - files: > - (?x) - ^providers/.*/src/airflow/providers/.*\.py$ - exclude: ^providers/standard/.*/.*\.py$ - - id: check-get-lineage-collector-providers - language: python - name: Check providers import hook lineage code from compat - description: Make sure you import from airflow.provider.common.compat.lineage.hook instead of - airflow.lineage.hook. - entry: ./scripts/ci/prek/check_airflow_imports.py - --pattern '^airflow\.lineage\.hook' - --message "Only TYPE_CHECKING imports from `airflow.lineage.hook` are allowed in providers." - --only_top_level - files: ^providers/.*/src/airflow/providers/.*\.py$ - exclude: ^providers/common/compat/src/airflow/providers/common/compat/.*\.py$ - id: check-decorated-operator-implements-custom-name name: Check @task decorator implements custom_operator_name language: python @@ -868,13 +687,6 @@ repos: language: python files: ^LICENSE$ pass_filenames: false - - id: check-aiobotocore-optional - name: Check if aiobotocore is an optional dependency only - entry: ./scripts/ci/prek/check_aiobotocore_optional.py - language: python - files: ^providers/.*/provider\.yaml$ - pass_filenames: true - require_serial: true - id: check-boring-cyborg-configuration name: Checks for Boring Cyborg configuration consistency language: python @@ -902,26 +714,6 @@ repos: files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$ require_serial: true pass_filenames: false - - id: lint-helm-chart - name: Lint Helm Chart - entry: ./scripts/ci/prek/lint_helm.py - language: python - pass_filenames: false - files: ^chart - require_serial: true - - id: validate-chart-annotations - name: Validate chart annotations - entry: ./scripts/ci/prek/validate_chart_annotations.py - language: python - pass_filenames: false - files: ^chart/Chart\.yaml$ - - id: kubeconform - name: Kubeconform check on our helm chart - entry: ./scripts/ci/prek/check_kubeconform.py - language: python - pass_filenames: false - files: ^chart - require_serial: true - id: shellcheck name: Check Shell scripts syntax correctness language: docker_image @@ -937,20 +729,6 @@ repos: entry: ./scripts/ci/prek/compile_ui_assets.py pass_filenames: false additional_dependencies: ['pnpm@9.7.1'] - - id: compile-fab-assets - name: Compile FAB provider assets - language: node - files: ^providers/fab/.*/www/ - entry: ./scripts/ci/prek/compile_provider_assets.py fab - pass_filenames: false - additional_dependencies: ['yarn@1.22.21'] - - id: compile-edge-assets - name: Compile Edge provider assets - language: node - files: ^providers/edge3/.*/www/ - entry: ./scripts/ci/prek/compile_provider_assets.py edge - pass_filenames: false - additional_dependencies: ['yarn@1.22.21'] - id: compile-ui-assets-dev name: Compile ui assets in dev mode (manual) language: node @@ -960,13 +738,6 @@ repos: entry: ./scripts/ci/prek/compile_ui_assets_dev.py pass_filenames: false additional_dependencies: ['pnpm@9.7.1'] - - id: check-providers-subpackages-init-file-exist - name: Provider subpackage init files are there - pass_filenames: false - always_run: true - entry: ./scripts/ci/prek/check_providers_subpackages_all_have_init.py - language: python - require_serial: true - id: check-integrations-list-consistent name: Sync integrations list with docs entry: ./scripts/ci/prek/check_integrations_list.py @@ -1062,34 +833,6 @@ repos: ^scripts/ci/docker-compose/gremlin/.| ^scripts/ci/docker-compose/.+-config\.ya?ml$ require_serial: true - - id: lint-json-schema - name: Lint chart/values.schema.json - entry: ./scripts/ci/prek/lint_json_schema.py - args: - - --spec-file - - chart/values_schema.schema.json - - chart/values.schema.json - language: python - pass_filenames: false - files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$ - require_serial: true - - id: update-vendored-in-k8s-json-schema - name: Vendor k8s definitions into values.schema.json - entry: ./scripts/ci/prek/vendor_k8s_json_schema.py - language: python - files: ^chart/values\.schema\.json$ - - id: lint-json-schema - name: Lint chart/values.yaml - entry: ./scripts/ci/prek/lint_json_schema.py - args: - - --enforce-defaults - - --spec-file - - chart/values.schema.json - - chart/values.yaml - language: python - pass_filenames: false - files: ^chart/values\.yaml$|^chart/values\.schema\.json$ - require_serial: true - id: lint-json-schema name: Lint config_templates/config.yml entry: ./scripts/ci/prek/lint_json_schema.py @@ -1127,13 +870,6 @@ repos: language: python pass_filenames: true files: ^airflow-core/src/airflow/.*\.py$ - - id: lint-chart-schema - name: Lint chart/values.schema.json file - entry: ./scripts/ci/prek/chart_schema.py - language: python - pass_filenames: false - files: ^chart/values\.schema\.json$ - require_serial: true - id: update-inlined-dockerfile-scripts name: Inline Dockerfile and Dockerfile.ci scripts entry: ./scripts/ci/prek/inline_scripts_in_docker.py @@ -1192,12 +928,6 @@ repos: pass_filenames: true files: ^airflow-core/docs/.*example-dags\.rst$|^docs/.*index\.rst$^airflow-core/docs/.*index\.rst$ always_run: true - - id: check-system-tests-tocs - name: Check that system tests is properly added - entry: ./scripts/ci/prek/check_system_tests_hidden_in_index.py - language: python - pass_filenames: true - files: ^providers/.*/docs/index\.rst$ - id: check-lazy-logging name: Check that all logging methods are lazy entry: ./scripts/ci/prek/check_lazy_logging.py @@ -1217,13 +947,6 @@ repos: language: python pass_filenames: true files: ^airflow-core/tests/.*\.py$ - - id: check-provider-docs-valid - name: Validate provider doc files - entry: ./scripts/ci/prek/check_provider_docs.py - language: python - files: ^providers/.*/provider\.yaml$|^.*/docs/.* - require_serial: true - pass_filenames: false - id: bandit name: bandit description: "Bandit is a tool for finding common security issues in Python code" @@ -1241,25 +964,7 @@ repos: - "B101,B301,B324,B403,B404,B603" - "--severity-level" - "high" # TODO: remove this line when we fix all the issues - - id: pylint - name: pylint - description: "Pylint is a static code analyser for Python 2 or 3." - entry: pylint - language: python - language_version: python3 - types: [python] - additional_dependencies: ['pylint==3.1.0'] - require_serial: true - files: ^airflow-core/src/airflow/.* - exclude: - airflow/example_dags/.* - args: - # Use pylint only for the specific check, which are not available into the ruff - - "--disable=all" - # W0133: "Exception statement has no effect" - # see: https://github.com/astral-sh/ruff/issues/10145 - - "--enable=W0133" - - id: check-fab-migrations + - id: check-no-fab-migrations language: pygrep name: Check no migration is done on FAB related table description: > @@ -1317,22 +1022,6 @@ repos: additional_dependencies: ['pnpm@9.7.1'] pass_filenames: true require_serial: true - - id: ts-compile-lint-edge-ui - name: Compile / format / lint edge UI - description: TS types generation / ESLint / Prettier new UI files in Edge Provider - language: node - files: | - (?x) - ^providers/edge3/src/airflow/providers/edge3/plugins/www/.*\.(js|ts|tsx|yaml|css|json)$| - ^providers/edge3/src/airflow/providers/edge3/openapi/v2-edge-generated.yaml$ - exclude: | - (?x) - ^providers/edge3/src/airflow/providers/edge3/plugins/www/node-modules/.*| - ^providers/edge3/src/airflow/providers/edge3/plugins/www/.pnpm-store - entry: ./scripts/ci/prek/ts_compile_lint_edge.py - additional_dependencies: ['pnpm@9.7.1'] - pass_filenames: true - require_serial: true ## ADD MOST PREK HOOK ABOVE THAT LINE # The below prek hooks are those requiring CI image to be built - id: mypy-dev @@ -1365,36 +1054,6 @@ repos: pass_filenames: false files: ^airflow-core/.*\.py$ require_serial: true - - id: mypy-providers - stages: ['pre-push'] - name: Run mypy for providers - language: python - entry: ./scripts/ci/prek/mypy.py - files: ^providers/.*\.py$ - require_serial: true - - id: mypy-providers - stages: ['manual'] - name: Run mypy for providers (manual) - language: python - entry: ./scripts/ci/prek/mypy_folder.py providers - pass_filenames: false - files: ^.*\.py$ - require_serial: true - - id: mypy-task-sdk - stages: ['pre-push'] - name: Run mypy for task-sdk - language: python - entry: ./scripts/ci/prek/mypy.py - files: ^task-sdk/.*\.py$ - require_serial: true - - id: mypy-task-sdk - stages: ['manual'] - name: Run mypy for task-sdk (manual) - language: python - entry: ./scripts/ci/prek/mypy_folder.py task-sdk - pass_filenames: false - files: ^.*\.py$ - require_serial: true - id: mypy-devel-common stages: ['pre-push'] name: Run mypy for devel-common @@ -1410,47 +1069,13 @@ repos: pass_filenames: false files: ^.*\.py$ require_serial: true - - id: mypy-airflow-ctl - stages: ['pre-push'] - name: Run mypy for airflow-ctl - language: python - entry: ./scripts/ci/prek/mypy.py - files: ^airflow-ctl/src/airflowctl/.*\.py$|^airflow-ctl/tests/.*\.py$ - exclude: .*generated.py - require_serial: true - - id: mypy-airflow-ctl - stages: ['manual'] - name: Run mypy for airflow-ctl (manual) - language: python - entry: ./scripts/ci/prek/mypy_folder.py airflow-ctl - pass_filenames: false - files: ^.*\.py$ - require_serial: true - id: generate-openapi-spec name: Generate the FastAPI API spec language: python entry: ./scripts/ci/prek/generate_openapi_spec.py pass_filenames: false - files: ^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$|^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ + files: ^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$ exclude: ^airflow-core/src/airflow/api_fastapi/execution_api/.* - - id: generate-openapi-spec-fab - name: Generate the FastAPI API spec for FAB - language: python - entry: ./scripts/ci/prek/generate_openapi_spec_providers.py fab - pass_filenames: false - files: ^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ - - id: generate-openapi-spec-edge - name: Generate the FastAPI API spec for Edge - language: python - entry: ./scripts/ci/prek/generate_openapi_spec_providers.py edge - pass_filenames: false - files: ^providers/edge3/src/airflow/providers/edge3/worker_api/.*\.py$ - - id: generate-openapi-spec-keycloak - name: Generate the FastAPI API spec for Keycloak - language: python - entry: ./scripts/ci/prek/generate_openapi_spec_providers.py keycloak - pass_filenames: false - files: ^providers/keycloak/src/airflow/providers/keycloak/auth_manager/.*\.py$ - id: check-i18n-json name: Check i18n files validity description: Check i18n files are valid json, have no TODOs, and auto-format them @@ -1458,13 +1083,6 @@ repos: files: ^airflow-core/src/airflow/ui/public/i18n/locales/.*\.json$ entry: ./scripts/ci/prek/check_i18n_json.py pass_filenames: false - - id: check-provider-yaml-valid - name: Validate provider.yaml files - entry: ./scripts/ci/prek/check_provider_yaml_files.py - language: python - files: ^providers/.*/provider\.yaml$ - exclude: ^providers/.*/.venv/.*$ - require_serial: true - id: check-template-fields-valid name: Check templated fields mapped in operators/sensors language: python @@ -1507,26 +1125,6 @@ repos: require_serial: true pass_filenames: false files: ^airflow-core/src/airflow/config_templates/config\.yml$ - - id: generate-airflowctl-help-images - name: Generate SVG from Airflow CTL Commands - entry: ./scripts/ci/prek/capture_airflowctl_help.py - language: python - pass_filenames: false - files: - ^airflow-ctl/src/airflowctl/ctl/cli_config.py$|airflow-ctl/src/airflowctl/api/operations.py|airflow-ctl/src/airflowctl/ctl/commands/.*\.py - - id: check-imports-in-providers - name: Check imports in providers - entry: ./scripts/ci/prek/check_imports_in_providers.py - language: python - files: ^providers/.*/src/airflow/providers/.*version_compat.*\.py$ - require_serial: true - - id: provider-version-compat - name: Check for correct version_compat imports in providers - entry: ./scripts/ci/prek/check_provider_version_compat.py - language: python - types: [python] - files: ^providers/.*/src/airflow/providers/.*\.py$ - require_serial: true - id: check-airflow-version-checks-in-core language: pygrep name: No AIRFLOW_V_* imports in airflow-core @@ -1542,7 +1140,6 @@ repos: ^airflow-core/tests/unit/core/test_configuration\.py$| ^airflow-core/tests/unit/models/test_renderedtifields\.py$| ^airflow-core/tests/unit/models/test_variable\.py$ - - id: check-sdk-imports name: Check for SDK imports in core files entry: ./scripts/ci/prek/check_sdk_imports.py @@ -1579,6 +1176,7 @@ repos: ^airflow-core/src/airflow/dag_processing/collection\.py$| ^airflow-core/src/airflow/dag_processing/manager\.py$| ^airflow-core/src/airflow/dag_processing/processor\.py$| + ^airflow-core/src/airflow/dag_processing/dagbag\.py$| ^airflow-core/src/airflow/datasets/metadata\.py$| ^airflow-core/src/airflow/exceptions\.py$| ^airflow-core/src/airflow/executors/local_executor\.py$| @@ -1590,9 +1188,10 @@ repos: ^airflow-core/src/airflow/models/__init__\.py$| ^airflow-core/src/airflow/models/asset\.py$| ^airflow-core/src/airflow/models/baseoperator\.py$| + ^airflow-core/src/airflow/models/callback\.py$| ^airflow-core/src/airflow/models/connection\.py$| ^airflow-core/src/airflow/models/dag\.py$| - ^airflow-core/src/airflow/models/dagbag\.py$| + ^airflow-core/src/airflow/models/dagbag.py$| ^airflow-core/src/airflow/models/dagrun\.py$| ^airflow-core/src/airflow/models/deadline\.py$| ^airflow-core/src/airflow/models/expandinput\.py$| diff --git a/.rat-excludes b/.rat-excludes index 129d7210f07b1..9890235c23c1d 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -19,6 +19,7 @@ .rat-excludes .stylelintignore .stylelintrc +.env .venv requirements requirements.txt @@ -176,9 +177,150 @@ auth_generated.py www-hash.txt # go setup files -go.mod -go.sum -mocks/* +**/go.mod +**/go.sum +**/protov1/* + +# go mocks +**/mocks/* + +# Generated protobuf files +.*proto +.*pb.go +.*_grpc.pb.go # Kubernetes env .env + +# SVG files +**/*.svg + +# Doc only change marker file +**/.latest-doc-only-change.txt +**/*-gen/* + + +# Redirects +**/redirects.txt + +# Ignore files + +**/.git-blame-ignore-revs +**/.gitattributes +**/.rat-excludes +**/.gitignore +**/.prettierignore +**/.prettierrc +**/.airflowignore +**/.airflowignore_glob + + +# Vendor includes +**/_vendor/ + +# Generated files +**/*-generated.yaml +**/*-generated.py +**/generated.py +**/generated/* +**/auth_generated.py + +# Lock files +**/pnpm-lock.yaml +**/yarn.lock +**/Chart.lock +**/uv.lock + +# Generated UI files +**/ui/index.html +**/ui/dev/index.html +**/ui/dist/index.html +**/_private_ui.yaml +**/dist/** +**/www/index.html + +# PNG files +**/*.png + +# CSV files +**/*.csv + +# LICENCE files +**/LICENCE*.txt +**/LICENSE*.txt + + +# Checksum files +**/*.sha256 +**/*.md5sum + +# Requirement files +**/requirements.txt + +# Hashes +**/command_hashes.txt +**/www-hash.txt + +# Spelling wordlist +**/spelling_wordlist.txt +**/dictionary.txt + +# Empty files +**/empty.txt + +# Script files +**/script +**/script.bteq +**/script_utf16.bteq + +# Reproducible build files +**/reproducible_build.yaml + +# Other files +**/test_notifier.txt +**/email.html +**/*.log +**/example_upload.txt +**/dummy.pdf +**/java_streaming_src/* +**/kube_config +**/prod_image_installed_providers.txt +**/text.txt +**/newsfragments/** +**/warnings.txt +**/rtd-deprecation/404.html +**/.env +**/*.jsonl + +# API files +**/_api/** +**/node_modules/** + +# Doc files +/docs/.latest-doc-only-change.txt +/docs/redirects.txt +/docs/integration-logos/*.svg +/docs/img/*.md5sum +/docs/img/*.svg + +# Log files +*.log + +# md5 sum files +.*\.md5sum + +# Generated files +*generated.* +/src/airflow/providers/keycloak/auth_manager/openapi/v2-keycloak-auth-manager-generated.yaml +/src/airflow/providers/edge3/plugins/www/* +/src/airflow/providers/edge3/openapi/v2-edge-generated.yaml +/src/airflow/providers/fab/auth_manager/api_fastapi/openapi/v2-fab-auth-manager-generated.yaml +/src/airflow/providers/fab/www/static/dist/* +/any/dag_id=dag_for_testing_redis_task_handler/run_id=test/task_id=task_for_testing_redis_log_handler/attempt=1.log +/src/airflow/providers/google/ads/.gitignore + +# Vendored-in code +/src/airflow/providers/google/_vendor/* + +# Git ignore file +.gitignore diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000000000..0e6d5fd64e4df --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "esbenp.prettier-vscode", + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000000..aeae611e1fe46 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,38 @@ +{ + "[javascript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.eslint": "explicit" + } + }, + "[javascriptreact]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.eslint": "explicit" + } + }, + "[typescript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.eslint": "explicit" + } + }, + "[typescriptreact]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.eslint": "explicit" + } + }, + "[json]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true + }, + "[jsonc]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true + }, +} diff --git a/Dockerfile b/Dockerfile index 0f7f5007b3a35..458d61f72eaa5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ ARG AIRFLOW_UID="50000" ARG AIRFLOW_USER_HOME_DIR=/home/airflow # latest released version here -ARG AIRFLOW_VERSION="3.1.1" +ARG AIRFLOW_VERSION="3.1.2" ARG BASE_IMAGE="debian:bookworm-slim" ARG AIRFLOW_PYTHON_VERSION="3.12.12" @@ -54,9 +54,9 @@ ARG AIRFLOW_PYTHON_VERSION="3.12.12" # You can swap comments between those two args to test pip from the main version # When you attempt to test if the version of `pip` from specified branch works for our builds # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` -ARG AIRFLOW_PIP_VERSION=25.2 +ARG AIRFLOW_PIP_VERSION=25.3 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_UV_VERSION=0.9.4 +ARG AIRFLOW_UV_VERSION=0.9.7 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" @@ -1116,6 +1116,7 @@ function install_from_sources() { --editable ./airflow-core --editable ./task-sdk --editable ./airflow-ctl \ --editable ./kubernetes-tests --editable ./docker-tests --editable ./helm-tests \ --editable ./task-sdk-tests \ + --editable ./airflow-ctl-tests \ --editable ./devel-common[all] --editable ./dev \ --group dev --group docs --group docs-gen --group leveldb" local -a projects_with_devel_dependencies diff --git a/Dockerfile.ci b/Dockerfile.ci index 74c7692031d8d..0927b49eb7ba3 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -870,6 +870,7 @@ function install_from_sources() { --editable ./airflow-core --editable ./task-sdk --editable ./airflow-ctl \ --editable ./kubernetes-tests --editable ./docker-tests --editable ./helm-tests \ --editable ./task-sdk-tests \ + --editable ./airflow-ctl-tests \ --editable ./devel-common[all] --editable ./dev \ --group dev --group docs --group docs-gen --group leveldb" local -a projects_with_devel_dependencies @@ -1653,10 +1654,10 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe # You can swap comments between those two args to test pip from the main version # When you attempt to test if the version of `pip` from specified branch works for our builds # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` -ARG AIRFLOW_PIP_VERSION=25.2 +ARG AIRFLOW_PIP_VERSION=25.3 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_UV_VERSION=0.9.4 -ARG AIRFLOW_PREK_VERSION="0.2.10" +ARG AIRFLOW_UV_VERSION=0.9.7 +ARG AIRFLOW_PREK_VERSION="0.2.12" # UV_LINK_MODE=copy is needed since we are using cache mounted from the host ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ diff --git a/README.md b/README.md index 9597bd82b7997..da7c29e17a3a6 100644 --- a/README.md +++ b/README.md @@ -25,15 +25,15 @@ | License | [![License](https://img.shields.io/:license-Apache%202-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0.txt) | | PyPI | [![PyPI version](https://badge.fury.io/py/apache-airflow.svg)](https://badge.fury.io/py/apache-airflow) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/apache-airflow.svg)](https://pypi.org/project/apache-airflow/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/apache-airflow)](https://pypi.org/project/apache-airflow/) | | Containers | [![Docker Pulls](https://img.shields.io/docker/pulls/apache/airflow.svg)](https://hub.docker.com/r/apache/airflow) [![Docker Stars](https://img.shields.io/docker/stars/apache/airflow.svg)](https://hub.docker.com/r/apache/airflow) [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/apache-airflow)](https://artifacthub.io/packages/search?repo=apache-airflow) | -| Community | [![Contributors](https://img.shields.io/github/contributors/apache/airflow)](https://github.com/apache/airflow/graphs/contributors) [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://s.apache.org/airflow-slack) ![Commit Activity](https://img.shields.io/github/commit-activity/m/apache/airflow) [![OSSRank](https://shields.io/endpoint?url=https://ossrank.com/shield/6)](https://ossrank.com/p/6) | +| Community | [![Contributors](https://img.shields.io/github/contributors/apache/airflow)](https://github.com/apache/airflow/graphs/contributors) [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://s.apache.org/airflow-slack) ![Commit Activity](https://img.shields.io/github/commit-activity/m/apache/airflow) [![LFX Health Score](https://insights.linuxfoundation.org/api/badge/health-score?project=apache-airflow)](https://insights.linuxfoundation.org/project/apache-airflow) | -| Version | Build Status | -|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Main | [![GitHub Build main](https://github.com/apache/airflow/actions/workflows/ci-amd.yml/badge.svg)](https://github.com/apache/airflow/actions) [![GitHub Build main](https://github.com/apache/airflow/actions/workflows/ci-arm.yml/badge.svg)](https://github.com/apache/airflow/actions) | -| 3.x | [![GitHub Build 3.0](https://github.com/apache/airflow/actions/workflows/ci-amd.yml/badge.svg?branch=v3-0-test)](https://github.com/apache/airflow/actions) [![GitHub Build 3.0](https://github.com/apache/airflow/actions/workflows/ci-arm.yml/badge.svg?branch=v3-0-test)](https://github.com/apache/airflow/actions) | -| 2.x | [![GitHub Build 2.11](https://github.com/apache/airflow/actions/workflows/ci.yml/badge.svg?branch=v2-11-test)](https://github.com/apache/airflow/actions) | +| Version | Build Status | +|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Main | [![GitHub Build main](https://github.com/apache/airflow/actions/workflows/ci-amd-arm.yml/badge.svg)](https://github.com/apache/airflow/actions) | +| 3.x | [![GitHub Build 3.1](https://github.com/apache/airflow/actions/workflows/ci-amd-arm.yml/badge.svg?branch=v3-1-test)](https://github.com/apache/airflow/actions) | +| 2.x | [![GitHub Build 2.11](https://github.com/apache/airflow/actions/workflows/ci.yml/badge.svg?branch=v2-11-test)](https://github.com/apache/airflow/actions) | @@ -99,14 +99,14 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (3.1.1) | -|------------|------------------------|------------------------| -| Python | 3.10, 3.11, 3.12, 3.13 | 3.10, 3.11, 3.12, 3.13 | -| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | -| Kubernetes | 1.30, 1.31, 1.32, 1.33 | 1.30, 1.31, 1.32, 1.33 | -| PostgreSQL | 13, 14, 15, 16, 17 | 13, 14, 15, 16, 17 | -| MySQL | 8.0, 8.4, Innovation | 8.0, 8.4, Innovation | -| SQLite | 3.15.0+ | 3.15.0+ | +| | Main version (dev) | Stable version (3.1.2) | +|------------|------------------------------|------------------------| +| Python | 3.10, 3.11, 3.12, 3.13 | 3.10, 3.11, 3.12, 3.13 | +| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | +| Kubernetes | 1.30, 1.31, 1.32, 1.33, 1.34 | 1.30, 1.31, 1.32, 1.33 | +| PostgreSQL | 13, 14, 15, 16, 17 | 13, 14, 15, 16, 17 | +| MySQL | 8.0, 8.4, Innovation | 8.0, 8.4, Innovation | +| SQLite | 3.15.0+ | 3.15.0+ | \* Experimental @@ -177,15 +177,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==3.1.1' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.1.1/constraints-3.10.txt" +pip install 'apache-airflow==3.1.2' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.1.2/constraints-3.10.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==3.1.1' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.1.1/constraints-3.10.txt" +pip install 'apache-airflow[postgres,google]==3.1.2' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.1.2/constraints-3.10.txt" ``` For information on installing provider distributions, check @@ -275,7 +275,7 @@ packages: Changing limits for versions of Airflow dependencies is not a breaking change on its own. * **Airflow Providers**: SemVer rules apply to changes in the particular provider's code only. SemVer MAJOR and MINOR versions for the packages are independent of the Airflow version. - For example, `google 4.1.0` and `amazon 3.0.6` providers can happily be installed + For example, `google 4.1.0` and `amazon 3.1.1` providers can happily be installed with `Airflow 2.1.2`. If there are limits of cross-dependencies between providers and Airflow packages, they are present in providers as `install_requires` limitations. We aim to keep backwards compatibility of providers with all previously released Airflow 2 versions but @@ -299,7 +299,7 @@ Apache Airflow version life cycle: | Version | Current Patch/Minor | State | First Release | Limited Maintenance | EOL/Terminated | |-----------|-----------------------|-----------|-----------------|-----------------------|------------------| -| 3 | 3.1.1 | Supported | Apr 22, 2025 | TBD | TBD | +| 3 | 3.1.2 | Supported | Apr 22, 2025 | TBD | TBD | | 2 | 2.11.0 | Supported | Dec 17, 2020 | Oct 22, 2025 | Apr 22, 2026 | | 1.10 | 1.10.15 | EOL | Aug 27, 2018 | Dec 17, 2020 | June 17, 2021 | | 1.9 | 1.9.0 | EOL | Jan 03, 2018 | Aug 27, 2018 | Aug 27, 2018 | @@ -432,7 +432,7 @@ might decide to add additional limits (and justify them with comment). Want to help build Apache Airflow? Check out our [contributors' guide](https://github.com/apache/airflow/blob/main/contributing-docs/README.rst) for a comprehensive overview of how to contribute, including setup instructions, coding standards, and pull request guidelines. -If you can't wait to contribute, and want to get started asap, check out the [contribution quickstart](https://github.com/apache/airflow/blob/main/contributing-docs/03_contributors_quick_start.rst) here! +If you can't wait to contribute, and want to get started asap, check out the [contribution quickstart](https://github.com/apache/airflow/blob/main/contributing-docs/03a_contributors_quick_start_beginners.rst) here! Official Docker (container) images for Apache Airflow are described in [images](https://github.com/apache/airflow/blob/main/dev/breeze/doc/ci/02_images.md). diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 3e62ce6da533e..e5fa249aa0176 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -24,6 +24,68 @@ .. towncrier release notes start +Airflow 3.1.2 (2025-11-05) +-------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +No significant changes. + +Bug Fixes +^^^^^^^^^ + +- Fix import error when upgrading structlog to 25.5.0+ (#57335) +- Fix connection retrieval in ``DagProcessorManager`` for bundle initialization (#57459) +- Fix incorrect task instance counts displayed in task group headers (#55670) +- Fix task retry execution after tasks are killed by external signals (#55767) +- Fix triggerer errors after Airflow 2 to 3 migration (#55884) +- Fix tasks unable to access ``triggering_user_name`` context variable (#56193) +- Fix outlet event extra data being empty in task instance success listener callbacks (#57031) +- UI: Fix panel button spacing and alignment issues (#57062) +- UI: Fix broken grid view links for tasks with retries (#57097) +- Fix DAG processor crash when renaming DAG tag case on MySQL (#57113) +- Fix iteration errors when using ``ObjectStoragePath`` (#57156) +- Fix auto-refresh not working on Required Actions page (#57207) +- Fix DAG processor crash by ignoring callbacks from other bundles (#57330) +- Fix asset name text overflow in DAG list view (#57363) +- Fix memory leak caused by repeated SSL context creation in API client (#57374) +- Fix performance issues loading DAG list page with many DAGs (#57444) +- Fix text selection jumping unexpectedly in log viewer (#57453) +- Fix DAG documentation pane not scroll-able when content is too long (#57518) +- Fix incorrect macro listings in template reference documentation (#57529) +- Fix Human-In-The-Loop operators failing when using notifiers (#57551) +- Fix n+1 query issues in XCom API endpoints (#57554) +- Fix n+1 query issues in Event Logs API endpoint (#57558) +- Fix n+1 query to fetch tags in the dags list page (#57570) +- Optimize database query to prevent "Out of sort memory" errors with many DAG versions (#57042) +- Optimize DAG list query for users with limited access (#57460) +- Optimize dynamic DAG updates to avoid loading large serialized DAGs (#57592) +- Reduce serialized DAG size by optimizing callback serialization in ``default_args`` (#57397) + +Miscellaneous +^^^^^^^^^^^^^ + +- UI: Improve global navigation visual design, interaction, and accessibility (#57565) +- UI: Add button to download all task logs at once (#56771) +- UI: Add timestamp column to ``XCom`` viewer and standardize task instance columns (#57447) +- UI: Improve highlighting of selected task instances and edges in grid view (#57560) +- Improve retry logic by migrating from ``retryhttp`` to ``tenacity`` library (#56762) +- Improve exception logging for task instance heartbeat failures (#57179) +- Add ``Content-Type`` header to Task SDK API requests (#57386) +- Log execution API server URL at task startup (#57409) +- Reduce log noise by changing "Connection not found" from error to debug level (#57548) +- Add ``task_display_name`` alias in event log API responses (#57609) +- Improve Pydantic model validation strictness in serialization (#57616) +- Fix systemd service files issues (#57231) + +Doc Only Changes +^^^^^^^^^^^^^^^^ +- Improve plugin system documentation for clarity and completeness (#57068) +- Improve clarity on api workers recommendation in docs (#57404) +- Fix ``instance_name`` in UI docs (#57523) +- Fix airflow macros list in template document (#57529) + Airflow 3.1.1 (2025-10-27) -------------------------- diff --git a/airflow-core/docs/administration-and-deployment/plugins.rst b/airflow-core/docs/administration-and-deployment/plugins.rst index 0e50686cd53f6..51f10859703f6 100644 --- a/airflow-core/docs/administration-and-deployment/plugins.rst +++ b/airflow-core/docs/administration-and-deployment/plugins.rst @@ -24,6 +24,9 @@ Airflow has a simple plugin manager built-in that can integrate external features to its core by simply dropping files in your ``$AIRFLOW_HOME/plugins`` folder. +Since Airflow 3.1, the plugin system supports new features such as React apps, FastAPI endpoints, +and middleware, making it easier to extend Airflow and build rich custom integrations. + The python modules in the ``plugins`` folder get imported, and **macros** and web **views** get integrated to Airflow's main collections and become available for use. @@ -48,7 +51,7 @@ Examples: * A set of tools to parse Hive logs and expose Hive metadata (CPU /IO / phases/ skew /...) * An anomaly detection framework, allowing people to collect metrics, set thresholds and alerts -* An auditing tool, helping understand who accesses what +* An auditing tool, helping to understand who accesses what * A config-driven SLA monitoring tool, allowing you to set monitored tables and at what time they should land, alert people, and expose visualizations of outages @@ -61,11 +64,24 @@ Airflow has many components that can be reused when building an application: * A metadata database to store your models * Access to your databases, and knowledge of how to connect to them * An array of workers that your application can push workload to -* Airflow is deployed, you can just piggy back on its deployment logistics +* Airflow is deployed, you can just piggyback on its deployment logistics * Basic charting capabilities, underlying libraries and abstractions .. _plugins:loading: +Available Building Blocks +------------------------- + +Airflow plugins can register the following components: + +* *External Views* – Add buttons/tabs linking to new pages in the UI. +* *React Apps* – Embed custom React apps inside the Airflow UI (new in Airflow 3.1). +* *FastAPI Apps* – Add custom API endpoints. +* *FastAPI Middlewares* – Intercept and modify API requests/responses. +* *Macros* – Define reusable Python functions available in DAG templates. +* *Operator Extra Links* – Add custom buttons in the task details view. +* *Timetables & Listeners* – Implement custom scheduling logic and event hooks. + When are plugins (re)loaded? ---------------------------- @@ -73,7 +89,7 @@ Plugins are by default lazily loaded and once loaded, they are never reloaded (e automatically loaded in Webserver). To load them at the start of each Airflow process, set ``[core] lazy_load_plugins = False`` in ``airflow.cfg``. -This means that if you make any changes to plugins and you want the webserver or scheduler to use that new +This means that if you make any changes to plugins, and you want the webserver or scheduler to use that new code you will need to restart those processes. However, it will not be reflected in new running tasks until after the scheduler boots. By default, task execution uses forking. This avoids the slowdown associated with creating a new Python interpreter @@ -151,6 +167,19 @@ additional initialization. Please note ``name`` inside this class must be specif Make sure you restart the webserver and scheduler after making changes to plugins so that they take effect. +Plugin Management Interface +--------------------------- + +Airflow 3.1 introduces a Plugin Management Interface, available under *Admin → Plugins* in the Airflow UI. +This page allows you to view installed plugins. + +... + +External Views +-------------- + +External views can also be embedded directly into the Airflow UI using iframes by providing a ``url_route`` value. +This allows you to render the view inline instead of opening it in a new browser tab. .. _plugin-example: diff --git a/airflow-core/docs/best-practices.rst b/airflow-core/docs/best-practices.rst index 183fe880c41c2..144f130447f40 100644 --- a/airflow-core/docs/best-practices.rst +++ b/airflow-core/docs/best-practices.rst @@ -310,7 +310,7 @@ Installing and Using ruff .. code-block:: bash - pip install "ruff>=0.14.1" + pip install "ruff>=0.14.3" 2. **Running ruff**: Execute ``ruff`` to check your Dags for potential issues: diff --git a/airflow-core/docs/howto/customize-ui.rst b/airflow-core/docs/howto/customize-ui.rst index 3681554f8aee8..68bfaba909c2a 100644 --- a/airflow-core/docs/howto/customize-ui.rst +++ b/airflow-core/docs/howto/customize-ui.rst @@ -32,11 +32,11 @@ distinguish between various installations of Airflow or simply amend the page te To make this change, simply: -1. Add the configuration option of ``instance_name`` under the ``[webserver]`` section inside ``airflow.cfg``: +1. Add the configuration option of ``instance_name`` under the ``[api]`` section inside ``airflow.cfg``: .. code-block:: - [webserver] + [api] instance_name = "DevEnv" @@ -45,7 +45,7 @@ To make this change, simply: .. code-block:: - AIRFLOW__WEBSERVER__INSTANCE_NAME = "DevEnv" + AIRFLOW__API__INSTANCE_NAME = "DevEnv" Screenshots diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml b/airflow-core/docs/howto/docker-compose/docker-compose.yaml index 2c2a614c9ef72..3892e04414358 100644 --- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml +++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml @@ -51,6 +51,8 @@ x-airflow-common: # and uncomment the "build" line below, Then run `docker-compose build` to build the images. image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:|version|} # build: . + env_file: + - ${ENV_FILE_PATH:-.env} environment: &airflow-common-env AIRFLOW__CORE__EXECUTOR: CeleryExecutor diff --git a/airflow-core/docs/img/change-site-title/default_instance_name_configuration.png b/airflow-core/docs/img/change-site-title/default_instance_name_configuration.png index 70fb653a510d0..3a814111c0e0c 100644 Binary files a/airflow-core/docs/img/change-site-title/default_instance_name_configuration.png and b/airflow-core/docs/img/change-site-title/default_instance_name_configuration.png differ diff --git a/airflow-core/docs/img/change-site-title/example_instance_name_configuration.png b/airflow-core/docs/img/change-site-title/example_instance_name_configuration.png index 45ec5e51d5a2a..5084842ad1015 100644 Binary files a/airflow-core/docs/img/change-site-title/example_instance_name_configuration.png and b/airflow-core/docs/img/change-site-title/example_instance_name_configuration.png differ diff --git a/airflow-core/docs/installation/supported-versions.rst b/airflow-core/docs/installation/supported-versions.rst index 34a04d5c7a23c..5d4ccb3c2b974 100644 --- a/airflow-core/docs/installation/supported-versions.rst +++ b/airflow-core/docs/installation/supported-versions.rst @@ -29,7 +29,7 @@ Apache Airflow® version life cycle: ========= ===================== ========= =============== ===================== ================ Version Current Patch/Minor State First Release Limited Maintenance EOL/Terminated ========= ===================== ========= =============== ===================== ================ -3 3.1.1 Supported Apr 22, 2025 TBD TBD +3 3.1.2 Supported Apr 22, 2025 TBD TBD 2 2.11.0 Supported Dec 17, 2020 Oct 22, 2025 Apr 22, 2026 1.10 1.10.15 EOL Aug 27, 2018 Dec 17, 2020 June 17, 2021 1.9 1.9.0 EOL Jan 03, 2018 Aug 27, 2018 Aug 27, 2018 diff --git a/airflow-core/docs/installation/upgrading_to_airflow3.rst b/airflow-core/docs/installation/upgrading_to_airflow3.rst index 57fd939ac8d40..d5a2057b6437a 100644 --- a/airflow-core/docs/installation/upgrading_to_airflow3.rst +++ b/airflow-core/docs/installation/upgrading_to_airflow3.rst @@ -61,7 +61,7 @@ Step 1: Take care of prerequisites ---------------------------------- - Make sure that you are on Airflow 2.7 or later. It is recommended to upgrade to latest 2.x and then to Airflow 3. -- Make sure that your Python version is in the supported list. Airflow 3.0.0 supports the following Python versions: Python 3.9, 3.10, 3.11 and 3.12. +- Make sure that your Python version is in the supported list. - Ensure that you are not using any features or functionality that have been :ref:`removed in Airflow 3`. diff --git a/airflow-core/docs/project.rst b/airflow-core/docs/project.rst index ade05e1321e81..3fea79274ef2b 100644 --- a/airflow-core/docs/project.rst +++ b/airflow-core/docs/project.rst @@ -59,6 +59,7 @@ Committers - Felix Uellendall (@feluelle) - Fokko Driesprong (@fokko) - Gopal Dirisala (@dirrao) +- Guan-Ming (Wesley) Chiu (@guan404ming) - Hitesh Shah (@hiteshs) - Hussein Awala (@hussein-awala) - Jakob Homan (@jghoman) diff --git a/airflow-core/docs/start.rst b/airflow-core/docs/start.rst index 2f934118e255d..3e4394d918988 100644 --- a/airflow-core/docs/start.rst +++ b/airflow-core/docs/start.rst @@ -68,7 +68,7 @@ This quick start guide will help you bootstrap an Airflow standalone instance on :substitutions: - AIRFLOW_VERSION=3.1.1 + AIRFLOW_VERSION=3.1.2 # Extract the version of Python you have installed. If you're currently using a Python version that is not supported by Airflow, you may want to set this manually. # See above for supported versions. diff --git a/airflow-core/docs/templates-ref.rst b/airflow-core/docs/templates-ref.rst index 1ce0b1852e752..0cbd8bf29c8ba 100644 --- a/airflow-core/docs/templates-ref.rst +++ b/airflow-core/docs/templates-ref.rst @@ -188,7 +188,7 @@ Variable Description Some Airflow specific macros are also defined: -.. automodule:: airflow.macros +.. automodule:: airflow.sdk.execution_time.macros :members: .. _pendulum.DateTime: https://pendulum.eustace.io/docs/#introduction diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml index 85e192c3ee5db..ac8d9f8eb7dfa 100644 --- a/airflow-core/pyproject.toml +++ b/airflow-core/pyproject.toml @@ -63,7 +63,7 @@ classifiers = [ ] # Version is defined in src/airflow/__init__.py and it is automatically synchronized by prek hook -version = "3.1.1" +version = "3.1.2" dependencies = [ "a2wsgi>=1.10.8", diff --git a/airflow-core/src/airflow/__init__.py b/airflow-core/src/airflow/__init__.py index d4ca8ab685e86..c4130e7d489b7 100644 --- a/airflow-core/src/airflow/__init__.py +++ b/airflow-core/src/airflow/__init__.py @@ -25,7 +25,7 @@ # lib.) This is required by some IDEs to resolve the import paths. __path__ = __import__("pkgutil").extend_path(__path__, __name__) -__version__ = "3.1.1" +__version__ = "3.1.2" import os diff --git a/airflow-core/src/airflow/api_fastapi/common/db/dags.py b/airflow-core/src/airflow/api_fastapi/common/db/dags.py index cefce66279883..f1104f839e3d4 100644 --- a/airflow-core/src/airflow/api_fastapi/common/db/dags.py +++ b/airflow-core/src/airflow/api_fastapi/common/db/dags.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING from sqlalchemy import func, select +from sqlalchemy.orm import selectinload from airflow.api_fastapi.common.db.common import ( apply_filters_to_select, @@ -32,14 +33,30 @@ from sqlalchemy.sql import Select -def generate_dag_with_latest_run_query(max_run_filters: list[BaseParam], order_by: SortParam) -> Select: - query = select(DagModel) +def generate_dag_with_latest_run_query( + max_run_filters: list[BaseParam], order_by: SortParam, *, dag_ids: set[str] | None = None +) -> Select: + """ + Generate a query to fetch DAGs with their latest run. - max_run_id_query = ( # ordering by id will not always be "latest run", but it's a simplifying assumption - select(DagRun.dag_id, func.max(DagRun.id).label("max_dag_run_id")) - .group_by(DagRun.dag_id) - .subquery(name="mrq") - ) + :param max_run_filters: List of filters to apply to the latest run + :param order_by: Sort parameter for ordering results + :param dag_ids: Optional set of DAG IDs to limit the query to. When provided, both the main + DAG query and the subquery for finding the latest runs will be filtered to + only these DAG IDs, improving performance when users have limited DAG access. + :return: SQLAlchemy Select statement + """ + query = select(DagModel).options(selectinload(DagModel.tags)) + + # Filter main query by dag_ids if provided + if dag_ids is not None: + query = query.where(DagModel.dag_id.in_(dag_ids or set())) + + # Also filter the subquery for finding latest runs + max_run_id_query_stmt = select(DagRun.dag_id, func.max(DagRun.id).label("max_dag_run_id")) + if dag_ids is not None: + max_run_id_query_stmt = max_run_id_query_stmt.where(DagRun.dag_id.in_(dag_ids or set())) + max_run_id_query = max_run_id_query_stmt.group_by(DagRun.dag_id).subquery(name="mrq") has_max_run_filter = False diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/event_logs.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/event_logs.py index d7355bc1294b1..8a1d429ad7027 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/event_logs.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/event_logs.py @@ -41,6 +41,9 @@ class EventLogResponse(BaseModel): dag_display_name: str | None = Field( validation_alias=AliasPath("dag_model", "dag_display_name"), default=None ) + task_display_name: str | None = Field( + validation_alias=AliasPath("task_instance", "task_display_name"), default=None + ) class EventLogCollectionResponse(BaseModel): diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dag_runs.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dag_runs.py index d99761f745753..6deaf958f4292 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dag_runs.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dag_runs.py @@ -18,6 +18,8 @@ from datetime import datetime +from pydantic import computed_field + from airflow.api_fastapi.core_api.base import BaseModel from airflow.utils.state import DagRunState @@ -33,3 +35,9 @@ class DAGRunLightResponse(BaseModel): start_date: datetime | None end_date: datetime | None state: DagRunState + + @computed_field + def duration(self) -> float | None: + if self.end_date and self.start_date: + return (self.end_date - self.start_date).total_seconds() + return None diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dags.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dags.py index 97221f6192d35..fc2fe3ed2d2aa 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dags.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dags.py @@ -18,16 +18,16 @@ from __future__ import annotations from airflow.api_fastapi.core_api.base import BaseModel -from airflow.api_fastapi.core_api.datamodels.dag_run import DAGRunResponse from airflow.api_fastapi.core_api.datamodels.dags import DAGResponse from airflow.api_fastapi.core_api.datamodels.hitl import HITLDetail +from airflow.api_fastapi.core_api.datamodels.ui.dag_runs import DAGRunLightResponse class DAGWithLatestDagRunsResponse(DAGResponse): """DAG with latest dag runs response serializer.""" asset_expression: dict | None - latest_dag_runs: list[DAGRunResponse] + latest_dag_runs: list[DAGRunLightResponse] pending_actions: list[HITLDetail] is_favorite: bool diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml index 344267c90a3a1..77d4ed18a8447 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml @@ -1347,137 +1347,24 @@ components: title: End Date state: $ref: '#/components/schemas/DagRunState' - type: object - required: - - id - - dag_id - - run_id - - logical_date - - run_after - - start_date - - end_date - - state - title: DAGRunLightResponse - description: DAG Run serializer for responses. - DAGRunResponse: - properties: - dag_run_id: - type: string - title: Dag Run Id - dag_id: - type: string - title: Dag Id - logical_date: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Logical Date - queued_at: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Queued At - start_date: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Start Date - end_date: - anyOf: - - type: string - format: date-time - - type: 'null' - title: End Date duration: anyOf: - type: number - type: 'null' title: Duration - data_interval_start: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Data Interval Start - data_interval_end: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Data Interval End - run_after: - type: string - format: date-time - title: Run After - last_scheduling_decision: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Last Scheduling Decision - run_type: - $ref: '#/components/schemas/DagRunType' - state: - $ref: '#/components/schemas/DagRunState' - triggered_by: - anyOf: - - $ref: '#/components/schemas/DagRunTriggeredByType' - - type: 'null' - triggering_user_name: - anyOf: - - type: string - - type: 'null' - title: Triggering User Name - conf: - anyOf: - - additionalProperties: true - type: object - - type: 'null' - title: Conf - note: - anyOf: - - type: string - - type: 'null' - title: Note - dag_versions: - items: - $ref: '#/components/schemas/DagVersionResponse' - type: array - title: Dag Versions - bundle_version: - anyOf: - - type: string - - type: 'null' - title: Bundle Version - dag_display_name: - type: string - title: Dag Display Name + readOnly: true type: object required: - - dag_run_id + - id - dag_id + - run_id - logical_date - - queued_at + - run_after - start_date - end_date - - duration - - data_interval_start - - data_interval_end - - run_after - - last_scheduling_decision - - run_type - state - - triggered_by - - triggering_user_name - - conf - - note - - dag_versions - - bundle_version - - dag_display_name - title: DAGRunResponse + - duration + title: DAGRunLightResponse description: DAG Run serializer for responses. DAGRunStates: properties: @@ -1662,7 +1549,7 @@ components: title: Asset Expression latest_dag_runs: items: - $ref: '#/components/schemas/DAGRunResponse' + $ref: '#/components/schemas/DAGRunLightResponse' type: array title: Latest Dag Runs pending_actions: @@ -1728,19 +1615,6 @@ components: so please ensure that their values always match the ones with the same name in TaskInstanceState.' - DagRunTriggeredByType: - type: string - enum: - - cli - - operator - - rest_api - - ui - - test - - timetable - - asset - - backfill - title: DagRunTriggeredByType - description: Class with TriggeredBy types for DagRun. DagRunType: type: string enum: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index 3dcb4924d96b1..8a631c1909678 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -10841,6 +10841,11 @@ components: - type: string - type: 'null' title: Dag Display Name + task_display_name: + anyOf: + - type: string + - type: 'null' + title: Task Display Name type: object required: - event_log_id diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/assets.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/assets.py index 839403352483e..a00739564f881 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/assets.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/assets.py @@ -182,6 +182,7 @@ def get_assets( subqueryload(AssetModel.scheduled_dags), subqueryload(AssetModel.producing_tasks), subqueryload(AssetModel.consuming_tasks), + subqueryload(AssetModel.aliases), ) ) @@ -304,7 +305,9 @@ def get_asset_events( session=session, ) - assets_event_select = assets_event_select.options(subqueryload(AssetEvent.created_dagruns)) + assets_event_select = assets_event_select.options( + subqueryload(AssetEvent.created_dagruns), joinedload(AssetEvent.asset) + ) assets_events = session.scalars(assets_event_select) return AssetEventCollectionResponse( diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dags.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dags.py index a50c1b35069d2..5c4a7662b396b 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dags.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dags.py @@ -137,6 +137,7 @@ def get_dags( last_dag_run_state, ], order_by=order_by, + dag_ids=readable_dags_filter.value, ) dags_select, total_entries = paginated_select( diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/event_logs.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/event_logs.py index 6e7603121e2ac..8909b5ff1bc64 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/event_logs.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/event_logs.py @@ -21,6 +21,7 @@ from fastapi import Depends, HTTPException, status from sqlalchemy import select +from sqlalchemy.orm import joinedload from airflow.api_fastapi.common.db.common import ( SessionDep, @@ -57,7 +58,9 @@ def get_event_log( event_log_id: int, session: SessionDep, ) -> EventLogResponse: - event_log = session.scalar(select(Log).where(Log.id == event_log_id)) + event_log = session.scalar( + select(Log).where(Log.id == event_log_id).options(joinedload(Log.task_instance)) + ) if event_log is None: raise HTTPException(status.HTTP_404_NOT_FOUND, f"The Event Log with id: `{event_log_id}` not found") return event_log @@ -125,7 +128,7 @@ def get_event_logs( event_pattern: Annotated[_SearchParam, Depends(search_param_factory(Log.event, "event_pattern"))], ) -> EventLogCollectionResponse: """Get all Event Logs.""" - query = select(Log) + query = select(Log).options(joinedload(Log.task_instance), joinedload(Log.dag_model)) event_logs_select, total_entries = paginated_select( statement=query, order_by=order_by, diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py index d31a5f8ba8e73..7169b34145a01 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/task_instances.py @@ -909,11 +909,23 @@ def patch_task_instance( for key, _ in data.items(): if key == "new_state": + # Create BulkTaskInstanceBody object with map_index field + bulk_ti_body = BulkTaskInstanceBody( + task_id=task_id, + map_index=map_index, + new_state=body.new_state, + note=body.note, + include_upstream=body.include_upstream, + include_downstream=body.include_downstream, + include_future=body.include_future, + include_past=body.include_past, + ) + _patch_task_instance_state( task_id=task_id, dag_run_id=dag_run_id, dag=dag, - task_instance_body=body, + task_instance_body=bulk_ti_body, data=data, session=session, ) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/xcom.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/xcom.py index 57b83f0dbb3a2..6aa86d09f22e7 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/xcom.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/xcom.py @@ -87,7 +87,7 @@ def get_xcom_entry( dag_ids=dag_id, map_indexes=map_index, limit=1, - ) + ).options(joinedload(XComModel.task), joinedload(XComModel.dag_run).joinedload(DR.dag_model)) # We use `BaseXCom.get_many` to fetch XComs directly from the database, bypassing the XCom Backend. # This avoids deserialization via the backend (e.g., from a remote storage like S3) and instead @@ -162,7 +162,7 @@ def get_xcom_entries( query = ( query.join(DR, and_(XComModel.dag_id == DR.dag_id, XComModel.run_id == DR.run_id)) .join(DagModel, DR.dag_id == DagModel.dag_id) - .options(joinedload(XComModel.dag_run).joinedload(DR.dag_model)) + .options(joinedload(XComModel.task), joinedload(XComModel.dag_run).joinedload(DR.dag_model)) ) if task_id != "~": @@ -285,7 +285,7 @@ def create_xcom_entry( XComModel.map_index == request_body.map_index, ) .limit(1) - .options(joinedload(XComModel.dag_run).joinedload(DR.dag_model)) + .options(joinedload(XComModel.task), joinedload(XComModel.dag_run).joinedload(DR.dag_model)) ) return XComResponseNative.model_validate(xcom) @@ -326,7 +326,7 @@ def update_xcom_entry( XComModel.map_index == patch_body.map_index, ) .limit(1) - .options(joinedload(XComModel.dag_run).joinedload(DR.dag_model)) + .options(joinedload(XComModel.task), joinedload(XComModel.dag_run).joinedload(DR.dag_model)) ) if not xcom_entry: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py index ff20874b50f0a..4ab981ab22331 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py @@ -52,7 +52,6 @@ filter_param_factory, ) from airflow.api_fastapi.common.router import AirflowRouter -from airflow.api_fastapi.core_api.datamodels.dag_run import DAGRunResponse from airflow.api_fastapi.core_api.datamodels.dags import DAGResponse from airflow.api_fastapi.core_api.datamodels.ui.dag_runs import DAGRunLightResponse from airflow.api_fastapi.core_api.datamodels.ui.dags import ( @@ -126,6 +125,7 @@ def get_dags( last_dag_run_state, ], order_by=order_by, + dag_ids=readable_dags_filter.value, ) dags_select, total_entries = paginated_select( @@ -183,7 +183,15 @@ def get_dags( recent_dag_runs_select = ( select( recent_runs_subquery.c.run_after, - DagRun, + DagRun.id, + DagRun.dag_id, + DagRun.run_id, + DagRun.end_date, + DagRun.logical_date, + DagRun.run_after, + DagRun.start_date, + DagRun.state, + DagRun.duration, ) .join( DagRun, @@ -205,7 +213,7 @@ def get_dags( # Fetch pending HITL actions for each Dag if we are not certain whether some of the Dag might contain HITL actions pending_actions_by_dag_id: dict[str, list[HITLDetail]] = {dag.dag_id: [] for dag in dags} - if has_pending_actions.value is not False: + if has_pending_actions.value: pending_actions_select = ( select( TaskInstance.dag_id, @@ -241,9 +249,8 @@ def get_dags( } for row in recent_dag_runs: - _, dag_run = row - dag_id = dag_run.dag_id - dag_run_response = DAGRunResponse.model_validate(dag_run) + dag_run_response = DAGRunLightResponse.model_validate(row) + dag_id = dag_run_response.dag_id dag_runs_by_dag_id[dag_id].latest_dag_runs.append(dag_run_response) return DAGWithLatestDagRunsCollectionResponse( diff --git a/airflow-core/src/airflow/api_fastapi/core_api/services/ui/grid.py b/airflow-core/src/airflow/api_fastapi/core_api/services/ui/grid.py index acf333c64a884..5b20511d31f47 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/services/ui/grid.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/services/ui/grid.py @@ -98,28 +98,26 @@ def _find_aggregates( "type": "mapped_task", "parent_id": parent_id, **_get_aggs_for_node(mapped_details), + "details": mapped_details, } return if isinstance(node, SerializedTaskGroup): - children = [] + children_details = [] for child in get_task_group_children_getter()(node): for child_node in _find_aggregates(node=child, parent_node=node, ti_details=ti_details): if child_node["parent_id"] == node_id: - children.append( - { - "state": child_node["state"], - "start_date": child_node["min_start_date"], - "end_date": child_node["max_end_date"], - } - ) + # Collect detailed task instance data from all children + if child_node.get("details"): + children_details.extend(child_node["details"]) yield child_node if node_id: yield { "task_id": node_id, "type": "group", "parent_id": parent_id, - **_get_aggs_for_node(children), + **_get_aggs_for_node(children_details), + "details": children_details, } return if isinstance(node, SerializedBaseOperator): @@ -128,5 +126,6 @@ def _find_aggregates( "type": "task", "parent_id": parent_id, **_get_aggs_for_node(details), + "details": details, } return diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py b/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py index 8708ed5122359..9f639b1e3911a 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py @@ -299,6 +299,7 @@ class DagRun(StrictBaseModel): run_type: DagRunType state: DagRunState conf: dict[str, Any] | None = None + triggering_user_name: str | None = None consumed_asset_events: list[AssetEventDagRunReference] diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py b/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py index efe9710c9ec74..a111b4dc91e96 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py @@ -29,10 +29,12 @@ from airflow.api_fastapi.execution_api.versions.v2025_10_27 import ( MakeDagRunConfNullable, ) +from airflow.api_fastapi.execution_api.versions.v2025_11_05 import AddTriggeringUserNameField bundle = VersionBundle( HeadVersion(), - Version("2025-10-27", MakeDagRunConfNullable), + Version("2025-11-05", MakeDagRunConfNullable), + Version("2025-10-10", AddTriggeringUserNameField), Version("2025-09-23", AddDagVersionIdField), Version( "2025-08-10", diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/versions/v2025_11_05.py b/airflow-core/src/airflow/api_fastapi/execution_api/versions/v2025_11_05.py new file mode 100644 index 0000000000000..ced2917afda05 --- /dev/null +++ b/airflow-core/src/airflow/api_fastapi/execution_api/versions/v2025_11_05.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from cadwyn import ResponseInfo, VersionChange, convert_response_to_previous_version_for, schema + +from airflow.api_fastapi.execution_api.datamodels.taskinstance import DagRun, TIRunContext + + +class AddTriggeringUserNameField(VersionChange): + """Add the `triggering_user_name` field to DagRun model.""" + + description = __doc__ + + instructions_to_migrate_to_previous_version = (schema(DagRun).field("triggering_user_name").didnt_exist,) + + @convert_response_to_previous_version_for(TIRunContext) # type: ignore[arg-type] + def remove_triggering_user_name_from_dag_run(response: ResponseInfo) -> None: # type: ignore[misc] + """Remove the `triggering_user_name` field from the dag_run object when converting to the previous version.""" + if "dag_run" in response.body and isinstance(response.body["dag_run"], dict): + response.body["dag_run"].pop("triggering_user_name", None) diff --git a/airflow-core/src/airflow/cli/commands/db_command.py b/airflow-core/src/airflow/cli/commands/db_command.py index a0a02ade87b24..ea3241320fdf8 100644 --- a/airflow-core/src/airflow/cli/commands/db_command.py +++ b/airflow-core/src/airflow/cli/commands/db_command.py @@ -81,8 +81,7 @@ def _get_version_revision(version: str, revision_heads_map: dict[str, str] | Non if current < wanted: return head - else: - return None + return None def run_db_migrate_command(args, command, revision_heads_map: dict[str, str]): diff --git a/airflow-core/src/airflow/config_templates/config.yml b/airflow-core/src/airflow/config_templates/config.yml index a7bd041362209..bb08fa49005b2 100644 --- a/airflow-core/src/airflow/config_templates/config.yml +++ b/airflow-core/src/airflow/config_templates/config.yml @@ -1389,9 +1389,9 @@ api: default: "8080" workers: description: | - Number of workers to run on the API server. Should be roughly equal to the number of - cpu cores available. If you need to scale the API server, consider deploying multiple API - servers instead of increasing the number of workers. + Number of workers to run on the API server. Should be roughly equal to the number of cpu cores + available. If you need to scale the API server, strongly consider deploying multiple API servers + instead of increasing the number of workers; See https://github.com/apache/airflow/issues/52270. version_added: ~ type: integer example: ~ diff --git a/airflow-core/src/airflow/configuration.py b/airflow-core/src/airflow/configuration.py index 042d6c84834bc..18b376ccba520 100644 --- a/airflow-core/src/airflow/configuration.py +++ b/airflow-core/src/airflow/configuration.py @@ -1717,8 +1717,7 @@ def _deprecated_value_is_set_in_config( deprecated_section_array = config.items(section=deprecated_section, raw=True) if any(key == deprecated_key for key, _ in deprecated_section_array): return True - else: - return False + return False @staticmethod def _deprecated_variable_is_set(deprecated_section: str, deprecated_key: str) -> bool: diff --git a/airflow-core/src/airflow/dag_processing/collection.py b/airflow-core/src/airflow/dag_processing/collection.py index 56de2fae758bf..a4e6af0337371 100644 --- a/airflow-core/src/airflow/dag_processing/collection.py +++ b/airflow-core/src/airflow/dag_processing/collection.py @@ -158,10 +158,32 @@ def calculate(cls, dags: dict[str, LazyDeserializedDAG], *, session: Session) -> def _update_dag_tags(tag_names: set[str], dm: DagModel, *, session: Session) -> None: orm_tags = {t.name: t for t in dm.tags} + tags_to_delete = [] for name, orm_tag in orm_tags.items(): if name not in tag_names: session.delete(orm_tag) - dm.tags.extend(DagTag(name=name, dag_id=dm.dag_id) for name in tag_names.difference(orm_tags)) + tags_to_delete.append(orm_tag) + + tags_to_add = tag_names.difference(orm_tags) + if tags_to_delete: + # Remove deleted tags from the collection to keep it in sync + for tag in tags_to_delete: + dm.tags.remove(tag) + + # Check if there's a potential case-only rename on MySQL (e.g., 'tag' -> 'TAG'). + # MySQL uses case-insensitive collation for the (name, dag_id) primary key by default, + # which can cause duplicate key errors when renaming tags with only case changes. + if session.bind.dialect.name == "mysql": + orm_tags_lower = {name.lower(): name for name in orm_tags} + has_case_only_change = any(tag.lower() in orm_tags_lower for tag in tags_to_add) + + if has_case_only_change: + # Force DELETE operations to execute before INSERT operations. + session.flush() + # Refresh the tags relationship from the database to reflect the deletions. + session.expire(dm, ["tags"]) + + dm.tags.extend(DagTag(name=name, dag_id=dm.dag_id) for name in tags_to_add) def _update_dag_owner_links(dag_owner_links: dict[str, str], dm: DagModel, *, session: Session) -> None: diff --git a/airflow-core/src/airflow/dag_processing/manager.py b/airflow-core/src/airflow/dag_processing/manager.py index df4f84d0ce548..c774aa29fc65d 100644 --- a/airflow-core/src/airflow/dag_processing/manager.py +++ b/airflow-core/src/airflow/dag_processing/manager.py @@ -250,6 +250,16 @@ def run(self): By processing them in separate processes, we can get parallelism and isolation from potentially harmful user code. """ + # TODO: Temporary until AIP-92 removes DB access from DagProcessorManager. + # The manager needs MetastoreBackend to retrieve connections from the database + # during bundle initialization (e.g., GitDagBundle.__init__ → GitHook needs git credentials). + # This marks the manager as "server" context so ensure_secrets_backend_loaded() provides + # MetastoreBackend instead of falling back to EnvironmentVariablesBackend only. + # Child parser processes explicitly override this by setting _AIRFLOW_PROCESS_CONTEXT=client + # in _parse_file_entrypoint() to prevent inheriting server privileges. + # Related: https://github.com/apache/airflow/pull/57459 + os.environ["_AIRFLOW_PROCESS_CONTEXT"] = "server" + self.register_exit_signals() self.log.info("Processing files using up to %s processes at a time ", self._parallelism) @@ -450,6 +460,7 @@ def _fetch_callbacks( callback_queue: list[CallbackRequest] = [] with prohibit_commit(session) as guard: + bundle_names = [bundle.name for bundle in self._dag_bundles] query = select(DbCallbackRequest) query = query.order_by(DbCallbackRequest.priority_weight.desc()).limit( self.max_callbacks_per_loop @@ -457,8 +468,11 @@ def _fetch_callbacks( query = with_row_locks(query, of=DbCallbackRequest, session=session, skip_locked=True) callbacks = session.scalars(query) for callback in callbacks: + req = callback.get_callback_request() + if req.bundle_name not in bundle_names: + continue try: - callback_queue.append(callback.get_callback_request()) + callback_queue.append(req) session.delete(callback) except Exception as e: self.log.warning("Error adding callback for execution: %s, %s", callback, e) diff --git a/airflow-core/src/airflow/dag_processing/processor.py b/airflow-core/src/airflow/dag_processing/processor.py index 9e7215784f200..ac30dc03358b7 100644 --- a/airflow-core/src/airflow/dag_processing/processor.py +++ b/airflow-core/src/airflow/dag_processing/processor.py @@ -171,6 +171,10 @@ def _pre_import_airflow_modules(file_path: str, log: FilteringBoundLogger) -> No def _parse_file_entrypoint(): + # Mark as client-side (runs user DAG code) + # Prevents inheriting server context from parent DagProcessorManager + os.environ["_AIRFLOW_PROCESS_CONTEXT"] = "client" + import structlog from airflow.sdk.execution_time import comms, task_runner diff --git a/airflow-core/src/airflow/jobs/triggerer_job_runner.py b/airflow-core/src/airflow/jobs/triggerer_job_runner.py index f5bc2d94d0601..617643c3f24ad 100644 --- a/airflow-core/src/airflow/jobs/triggerer_job_runner.py +++ b/airflow-core/src/airflow/jobs/triggerer_job_runner.py @@ -649,7 +649,13 @@ def update_triggers(self, requested_trigger_ids: set[int]): ) if new_trigger_orm.task_instance: log_path = render_log_fname(ti=new_trigger_orm.task_instance) - + if not new_trigger_orm.task_instance.dag_version_id: + # This is to handle 2 to 3 upgrade where TI.dag_version_id can be none + log.warning( + "TaskInstance associated with Trigger has no associated Dag Version, skipping the trigger", + ti_id=new_trigger_orm.task_instance.id, + ) + continue ser_ti = workloads.TaskInstance.model_validate( new_trigger_orm.task_instance, from_attributes=True ) diff --git a/airflow-core/src/airflow/models/log.py b/airflow-core/src/airflow/models/log.py index 9347e4a17536c..645c3ee6c7783 100644 --- a/airflow-core/src/airflow/models/log.py +++ b/airflow-core/src/airflow/models/log.py @@ -56,6 +56,14 @@ class Log(Base): primaryjoin="Log.dag_id == DagModel.dag_id", ) + task_instance = relationship( + "TaskInstance", + viewonly=True, + foreign_keys=[task_id], + primaryjoin="Log.task_id == TaskInstance.task_id", + lazy="noload", + ) + __table_args__ = ( Index("idx_log_dttm", dttm), Index("idx_log_event", event), diff --git a/airflow-core/src/airflow/models/serialized_dag.py b/airflow-core/src/airflow/models/serialized_dag.py index b6bbf67fae3c2..7137658825cf4 100644 --- a/airflow-core/src/airflow/models/serialized_dag.py +++ b/airflow-core/src/airflow/models/serialized_dag.py @@ -27,7 +27,7 @@ import sqlalchemy_jsonfield import uuid6 -from sqlalchemy import Column, ForeignKey, LargeBinary, String, exc, select, tuple_ +from sqlalchemy import Column, ForeignKey, LargeBinary, String, exc, select, tuple_, update from sqlalchemy.orm import backref, foreign, relationship from sqlalchemy.sql.expression import func, literal from sqlalchemy_utils import UUIDType @@ -427,14 +427,23 @@ def write_dag( # This is for dynamic DAGs that the hashes changes often. We should update # the serialized dag, the dag_version and the dag_code instead of a new version # if the dag_version is not associated with any task instances - latest_ser_dag = cls.get(dag.dag_id, session=session) - if TYPE_CHECKING: - assert latest_ser_dag is not None - # Update the serialized DAG with the new_serialized_dag - latest_ser_dag._data = new_serialized_dag._data - latest_ser_dag._data_compressed = new_serialized_dag._data_compressed - latest_ser_dag.dag_hash = new_serialized_dag.dag_hash - session.merge(latest_ser_dag) + + # Use direct UPDATE to avoid loading the full serialized DAG + result = session.execute( + update(cls) + .where(cls.dag_version_id == dag_version.id) + .values( + { + cls._data: new_serialized_dag._data, + cls._data_compressed: new_serialized_dag._data_compressed, + cls.dag_hash: new_serialized_dag.dag_hash, + } + ) + ) + + if result.rowcount == 0: + # No rows updated - serialized DAG doesn't exist + return False # The dag_version and dag_code may not have changed, still we should # do the below actions: # Update the latest dag version @@ -460,6 +469,15 @@ def write_dag( @classmethod def latest_item_select_object(cls, dag_id): + from airflow.settings import engine + + if engine.dialect.name == "mysql": + # Prevent "Out of sort memory" caused by large values in cls.data column for MySQL. + # Details in https://github.com/apache/airflow/pull/55589 + latest_item_id = ( + select(cls.id).where(cls.dag_id == dag_id).order_by(cls.created_at.desc()).limit(1) + ) + return select(cls).where(cls.id == latest_item_id) return select(cls).where(cls.dag_id == dag_id).order_by(cls.created_at.desc()).limit(1) @classmethod diff --git a/airflow-core/src/airflow/models/xcom.py b/airflow-core/src/airflow/models/xcom.py index 709d6bf69030c..0d3236eb408af 100644 --- a/airflow-core/src/airflow/models/xcom.py +++ b/airflow-core/src/airflow/models/xcom.py @@ -107,7 +107,7 @@ class XComModel(TaskInstanceDependencies): task = relationship( "TaskInstance", viewonly=True, - lazy="selectin", + lazy="noload", ) @classmethod diff --git a/airflow-core/src/airflow/serialization/serialized_objects.py b/airflow-core/src/airflow/serialization/serialized_objects.py index aed682f433744..626ead30132a5 100644 --- a/airflow-core/src/airflow/serialization/serialized_objects.py +++ b/airflow-core/src/airflow/serialization/serialized_objects.py @@ -2512,6 +2512,23 @@ def serialize_dag(cls, dag: DAG) -> dict: serialized_dag["has_on_success_callback"] = True if dag.has_on_failure_callback: serialized_dag["has_on_failure_callback"] = True + + # TODO: Move this logic to a better place -- ideally before serializing contents of default_args. + # There is some duplication with this and SerializedBaseOperator.partial_kwargs serialization. + # Ideally default_args goes through same logic as fields of SerializedBaseOperator. + if serialized_dag.get("default_args", {}): + default_args_dict = serialized_dag["default_args"][Encoding.VAR] + callbacks_to_remove = [] + for k, v in list(default_args_dict.items()): + if k in [ + f"on_{x}_callback" for x in ("execute", "failure", "success", "retry", "skipped") + ]: + if bool(v): + default_args_dict[f"has_{k}"] = True + callbacks_to_remove.append(k) + for k in callbacks_to_remove: + del default_args_dict[k] + return serialized_dag except SerializationError: raise diff --git a/airflow-core/src/airflow/serialization/typing.py b/airflow-core/src/airflow/serialization/typing.py index a6169b23a78d5..35166710b7810 100644 --- a/airflow-core/src/airflow/serialization/typing.py +++ b/airflow-core/src/airflow/serialization/typing.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +from dataclasses import is_dataclass from typing import Any @@ -29,4 +30,9 @@ def is_pydantic_model(cls: Any) -> bool: """ # __pydantic_fields__ is always present on Pydantic V2 models and is a dict[str, FieldInfo] # __pydantic_validator__ is an internal validator object, always set after model build - return hasattr(cls, "__pydantic_fields__") and hasattr(cls, "__pydantic_validator__") + # Check if it is not a dataclass to prevent detecting pydantic dataclasses as pydantic models + return ( + hasattr(cls, "__pydantic_fields__") + and hasattr(cls, "__pydantic_validator__") + and not is_dataclass(cls) + ) diff --git a/airflow-core/src/airflow/settings.py b/airflow-core/src/airflow/settings.py index 5a87384487641..9a54060ea5cfe 100644 --- a/airflow-core/src/airflow/settings.py +++ b/airflow-core/src/airflow/settings.py @@ -19,7 +19,7 @@ import atexit import functools -import json +import json as json_lib import logging import os import sys @@ -122,7 +122,7 @@ AsyncSession: Callable[..., SAAsyncSession] # The JSON library to use for DAG Serialization and De-Serialization -json = json +json = json_lib # Display alerts on the dashboard # Useful for warning about setup issues or announcing changes to end users diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index bdb71ac70f9bd..f9c93f976019f 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -3369,6 +3369,17 @@ export const $EventLogResponse = { } ], title: 'Dag Display Name' + }, + task_display_name: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Task Display Name' } }, type: 'object', @@ -6878,10 +6889,22 @@ export const $DAGRunLightResponse = { }, state: { '$ref': '#/components/schemas/DagRunState' + }, + duration: { + anyOf: [ + { + type: 'number' + }, + { + type: 'null' + } + ], + title: 'Duration', + readOnly: true } }, type: 'object', - required: ['id', 'dag_id', 'run_id', 'logical_date', 'run_after', 'start_date', 'end_date', 'state'], + required: ['id', 'dag_id', 'run_id', 'logical_date', 'run_after', 'start_date', 'end_date', 'state', 'duration'], title: 'DAGRunLightResponse', description: 'DAG Run serializer for responses.' } as const; @@ -7182,7 +7205,7 @@ export const $DAGWithLatestDagRunsResponse = { }, latest_dag_runs: { items: { - '$ref': '#/components/schemas/DAGRunResponse' + '$ref': '#/components/schemas/DAGRunLightResponse' }, type: 'array', title: 'Latest Dag Runs' diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index ee3c108e39c01..1396090691c41 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -877,6 +877,7 @@ export type EventLogResponse = { owner: string | null; extra: string | null; dag_display_name?: string | null; + task_display_name?: string | null; }; /** @@ -1742,6 +1743,7 @@ export type DAGRunLightResponse = { start_date: string | null; end_date: string | null; state: DagRunState; + readonly duration: number | null; }; /** @@ -1804,7 +1806,7 @@ export type DAGWithLatestDagRunsResponse = { asset_expression: { [key: string]: unknown; } | null; - latest_dag_runs: Array; + latest_dag_runs: Array; pending_actions: Array; is_favorite: boolean; /** diff --git a/airflow-core/src/airflow/ui/public/i18n/README.md b/airflow-core/src/airflow/ui/public/i18n/README.md index 18e4d44284dd9..60eb4485c5f3e 100644 --- a/airflow-core/src/airflow/ui/public/i18n/README.md +++ b/airflow-core/src/airflow/ui/public/i18n/README.md @@ -316,6 +316,18 @@ Adding missing translations (with `TODO: translate` prefix): uv run dev/i18n/check_translations_completeness.py --language --add-missing ``` +You can also remove extra translations from the language of your choice: + +```bash +uv run dev/i18n/check_translations_completeness.py --language --remove-extra +``` + +Or from all languages: + +```bash +uv run dev/i18n/check_translations_completeness.py --remove-extra +``` + The script is also added as a prek hook (manual) so that it can be run from within `prek` and CI: ```bash diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/ar/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/ar/components.json index 0b1c5d74aa9c8..df1b9d69d9b09 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/ar/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/ar/components.json @@ -120,7 +120,6 @@ "limitedList.clickToInteract": "انقر على علامة لتصفية Dags", "limitedList.clickToOpenFull": "انقر \"+{{count}} المزيد\" لعرض القائمة الكاملة", "limitedList.copyPasteText": "يمكنك نسخ ولصق النص أعلاه", - "limitedList.showingItems": "عرض {{count}} عنصرًا", "logs": { "file": "ملف", "location": "سطر {{line}} في {{name}}" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/ar/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/ar/dags.json index b7f8e60bbd8b5..12961f79af68d 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/ar/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/ar/dags.json @@ -20,8 +20,7 @@ "all": "الكل", "paused": "متوقف" }, - "runIdPatternFilter": "بحث تشغيلات الDag", - "triggeringUserNameFilter": "البحث حسب المستخدم صاحب الاطلاق" + "runIdPatternFilter": "بحث تشغيلات الDag" }, "ownerLink": "رابط المالك لـ{{owner}}", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/ca/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/ca/components.json index d5e7b5c197c12..b63b043d53f68 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/ca/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/ca/components.json @@ -92,7 +92,6 @@ "limitedList.clickToInteract": "Fes clic en una etiqueta per filtrar els Dags", "limitedList.clickToOpenFull": "Fes clic a \"+{{count}} més\" per veure la llista completa", "limitedList.copyPasteText": "Pots copiar i enganxar el text de dalt", - "limitedList.showingItems": "Mostrant {{count}} elements", "logs": { "file": "Fitxer", "location": "línia {{line}} a {{name}}" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/de/common.json b/airflow-core/src/airflow/ui/public/i18n/locales/de/common.json index e4d46a928f37e..fe14e8e0d6713 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/de/common.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/de/common.json @@ -78,11 +78,6 @@ "githubRepo": "GitHub Ablage", "restApiReference": "REST API Referenz" }, - "download": { - "download": "Herunterladen", - "hotkey": "d", - "tooltip": "Drücken Sie {{hotkey}}, um Protokolle herunterzuladen" - }, "duration": "Laufzeit", "endDate": "Enddatum", "error": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/de/hitl.json b/airflow-core/src/airflow/ui/public/i18n/locales/de/hitl.json index 48212bcd9d40b..56162417601cf 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/de/hitl.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/de/hitl.json @@ -1,8 +1,5 @@ { "filters": { - "body": "Nachricht", - "createdAtFrom": "Erstellt von ", - "createdAtTo": "Erstellt bis ", "response": { "all": "Alle", "pending": "Ausstehend", @@ -15,13 +12,11 @@ "requiredActionCount_other": "{{count}} offene Interaktionen", "requiredActionState": "Status der Interaktion", "response": { - "created": "Antwort erstellt um ", "error": "Senden der Antwort fehlgeschlagen", "optionsDescription": "Wählen Sie Ihre Optionen für diesen Task", "optionsLabel": "Optionen", "received": "Antwort empfangen um ", "respond": "Antworten", - "responded_by_user_name": "Beantwortet von (Benutzername)", "success": "{{taskId}} Interaktion erfolgreich", "title": "Erforderliche Interaktion - {{taskId}}" }, diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/el/common.json b/airflow-core/src/airflow/ui/public/i18n/locales/el/common.json index f3b725d02fd98..2870dda2c272d 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/el/common.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/el/common.json @@ -101,8 +101,6 @@ }, "filter": "Φίλτρο", "filters": { - "durationFrom": "Διάρκεια Από", - "durationTo": "Διάρκεια Έως", "logicalDateFrom": "Λογική Ημερομηνία Από", "logicalDateTo": "Λογική Ημερομηνία Έως", "runAfterFrom": "Εκτέλεση Μετά Από", @@ -163,7 +161,6 @@ }, "tooltip": "Πατήστε {{hotkey}} για κύλιση προς τα {{direction}}" }, - "seconds": "{{count}}δ", "security": { "actions": "Ενέργειες", "permissions": "Δικαιώματα", diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/en/common.json b/airflow-core/src/airflow/ui/public/i18n/locales/en/common.json index bcf197f1ddd62..211244bd68480 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/en/common.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/en/common.json @@ -78,6 +78,11 @@ "githubRepo": "GitHub Repo", "restApiReference": "REST API Reference" }, + "download": { + "download": "Download", + "hotkey": "d", + "tooltip": "Press {{hotkey}} to download logs" + }, "duration": "Duration", "endDate": "End Date", "error": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/es/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/es/components.json index 20f31033ee064..ad1edcd1b3967 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/es/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/es/components.json @@ -99,7 +99,6 @@ "limitedList.clickToInteract": "Haz clic en una etiqueta para filtrar Dags", "limitedList.clickToOpenFull": "Haz clic en \"+{{count}} más\" para ver la lista completa", "limitedList.copyPasteText": "Puedes copiar y pegar el texto de arriba", - "limitedList.showingItems": "Mostrando {{count}} elementos", "logs": { "file": "Archivo", "location": "línea {{line}} en {{name}}" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/es/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/es/dags.json index 7d93addccfc72..353ae56534a77 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/es/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/es/dags.json @@ -20,8 +20,7 @@ "all": "Todos", "paused": "Pausado" }, - "runIdPatternFilter": "Buscar Ejecuciones de Dag", - "triggeringUserNameFilter": "Buscar por usuario que activó" + "runIdPatternFilter": "Buscar Ejecuciones de Dag" }, "ownerLink": "Enlace de Propietario para {{owner}}", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/fr/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/fr/dags.json index cd6b2c2735631..b8a68745b81f5 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/fr/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/fr/dags.json @@ -20,8 +20,7 @@ "all": "Tous", "paused": "En pause" }, - "runIdPatternFilter": "Rechercher des exécutions de Dag", - "triggeringUserNameFilter": "Rechercher par utilisateur déclencheur" + "runIdPatternFilter": "Rechercher des exécutions de Dag" }, "ownerLink": "Lien du propriétaire pour {{owner}}", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/he/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/he/components.json index 424d5f675b14d..453f381980163 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/he/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/he/components.json @@ -99,7 +99,6 @@ "limitedList.clickToInteract": "לחץ על תגית כדי לסנן Dags", "limitedList.clickToOpenFull": "לחץ על \"+{{count}} נוספים\" כדי לפתוח את התצוגה המלאה", "limitedList.copyPasteText": "ניתן להעתיק ולהדביק את הטקסט שמעל", - "limitedList.showingItems": "מציג {{count}} פריטים", "logs": { "file": "קובץ", "location": "שורה {{line}} ב{{name}}" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/he/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/he/dags.json index 15729c5f1e866..1b3b06d4bf6b3 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/he/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/he/dags.json @@ -20,8 +20,7 @@ "all": "הכל", "paused": "מושהה" }, - "runIdPatternFilter": "חפש הרצת Dag", - "triggeringUserNameFilter": "חפש לפי שם המשתמש המפעיל" + "runIdPatternFilter": "חפש הרצת Dag" }, "ownerLink": "קישור בעלים ל-{{owner}}", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/hi/common.json b/airflow-core/src/airflow/ui/public/i18n/locales/hi/common.json index a506b9198b85c..1dcef4cb0cc1b 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/hi/common.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/hi/common.json @@ -25,7 +25,6 @@ "dag_other": "डैग्स", "dagDetails": { "catchup": "पकड़ना", - "concurrency": "समानांतरता", "dagRunTimeout": "डैग रन टाइमआउट", "defaultArgs": "डिफ़ॉल्ट तर्क", "description": "विवरण", @@ -188,8 +187,6 @@ "up_for_retry": "पुनः प्रयास के लिए", "upstream_failed": "अपस्ट्रीम विफल" }, - "switchToDarkMode": "डार्क मोड में स्विच करें", - "switchToLightMode": "लाइट मोड में स्विच करें", "table": { "completedAt": "पर पूर्ण", "createdAt": "पर बनाया", diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/hi/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/hi/components.json index 4efe768e9bebf..d3caf8c34c5eb 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/hi/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/hi/components.json @@ -92,7 +92,6 @@ "limitedList.clickToInteract": "टैग पर क्लिक करके Dags फ़िल्टर करें", "limitedList.clickToOpenFull": "\"+{{count}} और\" पर क्लिक करके पूरी सूची खोलें", "limitedList.copyPasteText": "आप ऊपर दिए गए पाठ को कॉपी और पेस्ट कर सकते हैं", - "limitedList.showingItems": "{{count}} आइटम दिखाए जा रहे हैं", "logs": { "file": "फ़ाइल", "location": "{{name}} में लाइन {{line}}" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/hi/dag.json b/airflow-core/src/airflow/ui/public/i18n/locales/hi/dag.json index 253ed9585407c..36e5d3adfca97 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/hi/dag.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/hi/dag.json @@ -40,7 +40,6 @@ "warning": "WARNING" }, "navigation": { - "jump": "जंप: Shift+{{arrow}}", "navigation": "नेवीगेशन: {{arrow}}", "toggleGroup": "ग्रुप टॉगल करें: Space" }, @@ -64,9 +63,7 @@ }, "panel": { "buttons": { - "options": "विकल्प", - "showGraph": "ग्राफ़ दिखाएं", - "showGrid": "ग्रिड दिखाएं" + "options": "विकल्प" }, "dagRuns": { "label": "डैग रन्स की संख्या" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/hi/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/hi/dags.json index 54cac11533683..74c112d0dfbcb 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/hi/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/hi/dags.json @@ -20,8 +20,7 @@ "all": "सभी", "paused": "रोका गया" }, - "runIdPatternFilter": "डैग रन्स खोजें", - "triggeringUserNameFilter": "ट्रिगर करने वाले उपयोगकर्ता द्वारा खोजें" + "runIdPatternFilter": "डैग रन्स खोजें" }, "ownerLink": "{{owner}} के लिए स्वामी लिंक", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/hu/common.json b/airflow-core/src/airflow/ui/public/i18n/locales/hu/common.json index f26a1540f08ac..084a73f3c448e 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/hu/common.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/hu/common.json @@ -189,8 +189,6 @@ "up_for_retry": "Újrapróbálkozásra vár", "upstream_failed": "Előfeltétel sikertelen" }, - "switchToDarkMode": "Váltás sötét módra", - "switchToLightMode": "Váltás világos módra", "table": { "completedAt": "Befejezve ekkor", "createdAt": "Létrehozva ekkor", diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/hu/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/hu/components.json index ded34869e9dac..014e8242b43a7 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/hu/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/hu/components.json @@ -92,7 +92,6 @@ "limitedList.clickToInteract": "Kattintson egy címkére a Dag-ok szűréséhez", "limitedList.clickToOpenFull": "Kattintson a \"+{{count}} további\" gombra a teljes nézethez", "limitedList.copyPasteText": "A fenti szöveg másolható és beilleszthető", - "limitedList.showingItems": "{{count}} elem megjelenítése", "logs": { "file": "Fájl", "location": "{{name}} fájl {{line}}. sora" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/hu/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/hu/dags.json index 93a925c7c4d7e..6394092f74718 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/hu/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/hu/dags.json @@ -20,8 +20,7 @@ "all": "Összes", "paused": "Szüneteltetett" }, - "runIdPatternFilter": "Dag futások keresése", - "triggeringUserNameFilter": "Keresés indító felhasználó alapján" + "runIdPatternFilter": "Dag futások keresése" }, "ownerLink": "Tulajdonosi hivatkozás: {{owner}}", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/it/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/it/components.json index e8903d951a2a8..3978d69905b7e 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/it/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/it/components.json @@ -106,7 +106,6 @@ "limitedList.clickToInteract": "Fai clic su un'etichetta per filtrare i Dag", "limitedList.clickToOpenFull": "Fai clic su \"+{{count}} altro\" per visualizzare tutto", "limitedList.copyPasteText": "Puoi copiare e incollare il testo sopra", - "limitedList.showingItems": "Visualizzazione di {{count}} elementi", "logs": { "file": "File", "location": "linea {{line}} in {{name}}" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/it/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/it/dags.json index 0cb4eac8ac761..6c6ab552ddc9c 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/it/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/it/dags.json @@ -20,8 +20,7 @@ "all": "Tutti", "paused": "In Pausa" }, - "runIdPatternFilter": "Cercare Dag Runs", - "triggeringUserNameFilter": "Cercare per Utente che ha Attivato" + "runIdPatternFilter": "Cercare Dag Runs" }, "ownerLink": "Link dell'Owner per {{owner}}", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/ko/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/ko/components.json index e4dfd34a9f500..190a430bd44e9 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/ko/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/ko/components.json @@ -90,7 +90,6 @@ "limitedList.clickToInteract": "태그를 클릭하여 Dag를 필터링하세요", "limitedList.clickToOpenFull": "\"+{{count}}개 더 보기\"를 클릭하여 전체 보기 열기", "limitedList.copyPasteText": "위의 텍스트를 복사하여 붙여넣을 수 있습니다", - "limitedList.showingItems": "{{count}}개 항목 표시 중", "logs": { "file": "파일", "location": "{{name}}의 {{line}}번째 줄" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/ko/dag.json b/airflow-core/src/airflow/ui/public/i18n/locales/ko/dag.json index 554d8d5263e1a..fc722b1f826ae 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/ko/dag.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/ko/dag.json @@ -95,9 +95,7 @@ "buttons": { "options": "옵션", "showGantt": "간트 차트 보기", - "showGraph": "그래프 보기", "showGraphShortcut": "그래프 보기 (g 키)", - "showGrid": "그리드 보기", "showGridShortcut": "그리드 보기 (g 키)" }, "dagRuns": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/nl/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/nl/components.json index 83785a7112a41..9997e06717296 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/nl/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/nl/components.json @@ -92,7 +92,6 @@ "limitedList.clickToInteract": "Klik op een label om Dags te filteren", "limitedList.clickToOpenFull": "Klik op \"+{{count}} meer\" om de volledige lijst te openen", "limitedList.copyPasteText": "Je kunt de bovenstaande tekst kopiëren en plakken", - "limitedList.showingItems": "{{count}} items weergegeven", "logs": { "file": "Bestand", "location": "regel {{line}} in {{name}}" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/nl/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/nl/dags.json index b57b2290a9232..23fdb3bc2d480 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/nl/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/nl/dags.json @@ -20,8 +20,7 @@ "all": "Alles", "paused": "Gepauzeerd" }, - "runIdPatternFilter": "Zoek Dag Runs", - "triggeringUserNameFilter": "Zoek op Triggering User" + "runIdPatternFilter": "Zoek Dag Runs" }, "ownerLink": "Eigenaarslink voor {{owner}}", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/pl/common.json b/airflow-core/src/airflow/ui/public/i18n/locales/pl/common.json index eaafe9c4b5a6b..04ca807223bc1 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/pl/common.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/pl/common.json @@ -113,8 +113,6 @@ }, "filter": "Filtr", "filters": { - "durationFrom": "Czas trwania od", - "durationTo": "Czas trwania do", "logicalDateFrom": "Data logiczna od", "logicalDateTo": "Data logiczna do", "runAfterFrom": "Uruchom po (od)", diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/pl/hitl.json b/airflow-core/src/airflow/ui/public/i18n/locales/pl/hitl.json index c5023a8ad121d..428f1313ce1ed 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/pl/hitl.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/pl/hitl.json @@ -1,8 +1,5 @@ { "filters": { - "body": "Treść", - "createdAtFrom": "Utworzono od", - "createdAtTo": "Utworzono do", "response": { "all": "Wszystkie", "pending": "Oczekujące", diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/pt/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/pt/components.json index ae83855677a6c..5b30826d8b716 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/pt/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/pt/components.json @@ -106,7 +106,6 @@ "limitedList.clickToInteract": "Clique em uma etiqueta para filtrar os Dags", "limitedList.clickToOpenFull": "Clique em \"+{{count}} mais\" para ver a lista completa", "limitedList.copyPasteText": "Você pode copiar e colar o texto acima", - "limitedList.showingItems": "Exibindo {{count}} itens", "logs": { "file": "Arquivo", "location": "linha {{line}} em {{name}}" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/pt/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/pt/dags.json index 683cf090a8b15..6ac6422c19ed1 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/pt/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/pt/dags.json @@ -20,8 +20,7 @@ "all": "Todos", "paused": "Pausado" }, - "runIdPatternFilter": "Pesquisar Execuções de DAG", - "triggeringUserNameFilter": "Pesquisar por Usuário que Disparou" + "runIdPatternFilter": "Pesquisar Execuções de DAG" }, "ownerLink": "Link do Proprietário para {{owner}}", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/th/common.json b/airflow-core/src/airflow/ui/public/i18n/locales/th/common.json index 543603805bdb4..f55f65988278d 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/th/common.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/th/common.json @@ -101,8 +101,6 @@ }, "filter": "ตัวกรอง", "filters": { - "durationFrom": "ระยะเวลาตั้งแต่", - "durationTo": "ระยะเวลาถึง", "logicalDateFrom": "วันที่ตามกำหนดทำงานตั้งแต่", "logicalDateTo": "วันที่ตามกำหนดทำงานถึง", "runAfterFrom": "ทำงานตั้งแต่", diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/tr/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/tr/components.json index 69d80a7672ba9..0a32afc28b301 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/tr/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/tr/components.json @@ -92,7 +92,6 @@ "limitedList.clickToInteract": "Bir etikete tıklayarak Dag'leri filtreleyin", "limitedList.clickToOpenFull": "\"+{{count}} daha\" tıklayarak tam görünümü açın", "limitedList.copyPasteText": "Yukarıdaki metni kopyalayıp yapıştırabilirsiniz", - "limitedList.showingItems": "{{count}} öğe gösteriliyor", "logs": { "file": "Dosya", "location": "{{name}} içinde satır {{line}}" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/tr/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/tr/dags.json index 2974a4d4f4860..8c6d021305232 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/tr/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/tr/dags.json @@ -20,8 +20,7 @@ "all": "Tümü", "paused": "Duraklatılmış" }, - "runIdPatternFilter": "Dag Çalıştırmalarında Ara", - "triggeringUserNameFilter": "Tetikleyen Kullanıcıya Göre Ara" + "runIdPatternFilter": "Dag Çalıştırmalarında Ara" }, "ownerLink": "{{owner}} için sahip bağlantısı", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/zh-CN/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/zh-CN/components.json index e7255110e8f3b..652011da2bea8 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/zh-CN/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/zh-CN/components.json @@ -90,7 +90,6 @@ "limitedList.clickToInteract": "点击标签以筛选 DAGs", "limitedList.clickToOpenFull": "点击 \"+{{count}} 更多\" 打开完整视图", "limitedList.copyPasteText": "你可以复制并粘贴上方文本", - "limitedList.showingItems": "显示 {{count}} 项", "logs": { "file": "文件", "location": "第 {{line}} 行，位于 {{name}}" diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/zh-CN/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/zh-CN/dags.json index b8e4f0929d4e0..90ba4ff9588cd 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/zh-CN/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/zh-CN/dags.json @@ -20,8 +20,7 @@ "all": "全部", "paused": "暂停" }, - "runIdPatternFilter": "搜索 Dag 执行", - "triggeringUserNameFilter": "搜索触发用户名称" + "runIdPatternFilter": "搜索 Dag 执行" }, "ownerLink": "拥有者 {{owner}} 的地址", "runAndTaskActions": { diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/zh-TW/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/zh-TW/components.json index c655e241acda4..bd4fe614be232 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/zh-TW/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/zh-TW/components.json @@ -90,7 +90,6 @@ "limitedList.clickToInteract": "點擊標籤以篩選 Dags", "limitedList.clickToOpenFull": "點擊 \"+{{count}} 更多\" 以開啟完整檢視", "limitedList.copyPasteText": "你可以複製並貼上上方文字", - "limitedList.showingItems": "顯示 {{count}} 個項目", "logs": { "file": "檔案", "location": "第 {{line}} 行，位於 {{name}}" diff --git a/airflow-core/src/airflow/ui/src/components/Graph/Edge.tsx b/airflow-core/src/airflow/ui/src/components/Graph/Edge.tsx index 156384d11384a..a337572797da0 100644 --- a/airflow-core/src/airflow/ui/src/components/Graph/Edge.tsx +++ b/airflow-core/src/airflow/ui/src/components/Graph/Edge.tsx @@ -27,7 +27,7 @@ import type { EdgeData } from "./reactflowUtils"; type Props = EdgeType; const CustomEdge = ({ data }: Props) => { - const [strokeColor] = useToken("colors", ["border.inverted"]); + const [strokeColor, blueColor] = useToken("colors", ["border.inverted", "blue.500"]); if (data === undefined) { return undefined; @@ -60,9 +60,9 @@ const CustomEdge = ({ data }: Props) => { point.x} y={(point: ElkPoint) => point.y} /> diff --git a/airflow-core/src/airflow/ui/src/components/Graph/TaskNode.tsx b/airflow-core/src/airflow/ui/src/components/Graph/TaskNode.tsx index 803a19959ea72..2674d32a953a7 100644 --- a/airflow-core/src/airflow/ui/src/components/Graph/TaskNode.tsx +++ b/airflow-core/src/airflow/ui/src/components/Graph/TaskNode.tsx @@ -75,7 +75,7 @@ export const TaskNode = ({ // Alternate background color for nested open groups bg={isOpen && depth !== undefined && depth % 2 === 0 ? "bg.muted" : "bg"} borderColor={ - taskInstance?.state ? `${taskInstance.state}.solid` : isSelected ? "border.inverted" : "border" + isSelected ? "blue.500" : taskInstance?.state ? `${taskInstance.state}.solid` : "border" } borderRadius={5} borderWidth={isSelected ? 4 : 2} diff --git a/airflow-core/src/airflow/ui/src/components/renderStructuredLog.tsx b/airflow-core/src/airflow/ui/src/components/renderStructuredLog.tsx index a5707cdb1ceb0..f810f57554bd9 100644 --- a/airflow-core/src/airflow/ui/src/components/renderStructuredLog.tsx +++ b/airflow-core/src/airflow/ui/src/components/renderStructuredLog.tsx @@ -47,6 +47,7 @@ type RenderStructuredLogProps = { logLevelFilters?: Array; logLink: string; logMessage: string | StructuredLogMessage; + renderingMode?: "jsx" | "text"; showSource?: boolean; showTimestamp?: boolean; sourceFilters?: Array; @@ -107,17 +108,22 @@ const addAnsiWithLinks = (line: string) => { const sourceFields = ["logger", "chan", "lineno", "filename", "loc"]; -export const renderStructuredLog = ({ +const renderStructuredLogImpl = ({ index, logLevelFilters, logLink, logMessage, + renderingMode = "jsx", showSource = true, showTimestamp = true, sourceFilters, translate, -}: RenderStructuredLogProps) => { +}: RenderStructuredLogProps): JSX.Element | string => { if (typeof logMessage === "string") { + if (renderingMode === "text") { + return logMessage; + } + return ( {addAnsiWithLinks(logMessage)} @@ -147,22 +153,32 @@ export const renderStructuredLog = ({ } if (Boolean(timestamp) && showTimestamp) { - elements.push("[",