diff --git a/.github/ISSUE_TEMPLATE/airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_bug_report.yml index 25fb14cba10e4..bd038baf6346b 100644 --- a/.github/ISSUE_TEMPLATE/airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_bug_report.yml @@ -25,8 +25,8 @@ body: the latest release or main to see if the issue is fixed before reporting it. multiple: false options: - - "2.10.5" - - "3.0.0" + - "3.0.1" + - "2.11.0" - "main (development)" - "Other Airflow 2 version (please specify below)" validations: diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml index 3a427ee12519e..5289389930c15 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-pre-commit/action.yml @@ -24,7 +24,7 @@ inputs: default: "3.9" uv-version: description: 'uv version to use' - default: "0.7.2" # Keep this comment to allow automatic replacement of uv version + default: "0.7.8" # Keep this comment to allow automatic replacement of uv version pre-commit-version: description: 'pre-commit version to use' default: "4.2.0" # Keep this comment to allow automatic replacement of pre-commit version diff --git a/.github/actions/migration_tests/action.yml b/.github/actions/migration_tests/action.yml index ed71e21407d10..0198cbf472b0b 100644 --- a/.github/actions/migration_tests/action.yml +++ b/.github/actions/migration_tests/action.yml @@ -24,12 +24,13 @@ runs: - name: "Test migration file 2 to 3 migration: ${{env.BACKEND}}" shell: bash run: | - breeze shell "${{ env.AIRFLOW_2_CMD }}" --use-airflow-version 2.10.5 --answer y && - breeze shell "${{ env.AIRFLOW_3_CMD }}" --no-db-cleanup + breeze shell "${{ env.AIRFLOW_2_CMD }}" --use-airflow-version 2.11.0 --answer y && + breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${{env.DB_MANGERS}} + ${{ env.AIRFLOW_3_CMD }}" --no-db-cleanup env: COMPOSE_PROJECT_NAME: "docker-compose" - AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS: "airflow.providers.fab.auth_manager.models.db.FABDBManager" DB_RESET: "false" + DB_MANAGERS: "airflow.providers.fab.auth_manager.models.db.FABDBManager" AIRFLOW_2_CMD: >- airflow db reset --skip-init -y && airflow db migrate --to-revision heads @@ -46,12 +47,13 @@ runs: - name: "Test ORM migration 2 to 3: ${{env.BACKEND}}" shell: bash run: > - breeze shell "${{ env.AIRFLOW_2_CMD }}" --use-airflow-version 2.10.5 --answer y && - breeze shell "${{ env.AIRFLOW_3_CMD }}" --no-db-cleanup + breeze shell "${{ env.AIRFLOW_2_CMD }}" --use-airflow-version 2.11.0 --answer y && + breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${{env.DB_MANGERS}} + ${{ env.AIRFLOW_3_CMD }}" --no-db-cleanup env: COMPOSE_PROJECT_NAME: "docker-compose" - AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS: "airflow.providers.fab.auth_manager.models.db.FABDBManager" DB_RESET: "false" + DB_MANAGERS: "airflow.providers.fab.auth_manager.models.db.FABDBManager" AIRFLOW_2_CMD: >- airflow db reset -y AIRFLOW_3_CMD: >- @@ -67,13 +69,14 @@ runs: - name: "Test ORM migration ${{env.BACKEND}}" shell: bash run: > - breeze shell "airflow db reset -y && + breeze shell "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${{env.DB_MANAGERS}} && + airflow db reset -y && airflow db migrate --to-revision heads && airflow db downgrade -n 2.7.0 -y && airflow db migrate" env: COMPOSE_PROJECT_NAME: "docker-compose" - AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS: "airflow.providers.fab.auth_manager.models.db.FABDBManager" + DB_MANAGERS: "airflow.providers.fab.auth_manager.models.db.FABDBManager" - name: "Bring compose down again" shell: bash run: breeze down @@ -82,12 +85,14 @@ runs: - name: "Test offline migration ${{env.BACKEND}}" shell: bash run: > - breeze shell "airflow db reset -y && + breeze shell + "export AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS=${{env.DB_MANAGERS}} && + airflow db reset -y && airflow db downgrade -n 2.7.0 -y && airflow db migrate -s" env: COMPOSE_PROJECT_NAME: "docker-compose" - AIRFLOW__DATABASE__EXTERNAL_DB_MANAGERS: "airflow.providers.fab.auth_manager.models.db.FABDBManager" + DB_MANAGERS: "airflow.providers.fab.auth_manager.models.db.FABDBManager" if: env.BACKEND != 'sqlite' - name: "Bring any containers left down" shell: bash diff --git a/.github/actions/post_tests_success/action.yml b/.github/actions/post_tests_success/action.yml index b7b00a6fc0df3..865f1e4857cb2 100644 --- a/.github/actions/post_tests_success/action.yml +++ b/.github/actions/post_tests_success/action.yml @@ -44,7 +44,7 @@ runs: mkdir ./files/coverage-reports mv ./files/coverage*.xml ./files/coverage-reports/ || true - name: "Upload all coverage reports to codecov" - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4 env: CODECOV_TOKEN: ${{ inputs.codecov-token }} if: env.ENABLE_COVERAGE == 'true' && env.TEST_TYPES != 'Helm' && inputs.python-version != '3.12' diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 3254254a86516..4a2685cc4ebd8 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -38,14 +38,19 @@ outputs: runs: using: "composite" steps: - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh shell: bash - name: "Install Breeze" uses: ./.github/actions/breeze with: use-uv: ${{ inputs.use-uv }} id: breeze + - name: "Check free space" + shell: bash + run: | + echo "Checking free space!" + df -H - name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}" uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468 with: diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 296e58fa23ad2..9e7edaee0ee85 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -138,8 +138,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 489b2967af3d1..a2e9deed6df5c 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -72,7 +72,6 @@ jobs: python-versions: "[ '${{ inputs.default-python-version }}' ]" default-python-version: ${{ inputs.default-python-version }} branch: ${{ inputs.default-branch }} - use-uv: "false" upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} @@ -88,7 +87,6 @@ jobs: python-versions: "[ '${{ inputs.default-python-version }}' ]" default-python-version: ${{ inputs.default-python-version }} branch: ${{ inputs.default-branch }} - use-uv: "false" upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} @@ -113,8 +111,6 @@ jobs: with: fetch-depth: 2 persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 5d931b001865a..52983ad65d559 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -76,8 +76,8 @@ jobs: # Need to fetch all history for selective checks tests fetch-depth: 0 persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -97,8 +97,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: Setup pnpm uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # v4.0.0 with: @@ -194,8 +194,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -249,8 +249,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -322,8 +322,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 4859adf0dfec1..e67351d64108f 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -137,8 +137,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: Fetch incoming commit ${{ github.sha }} with its parent uses: actions/checkout@v4 with: @@ -291,7 +291,8 @@ jobs: packages: read if: > needs.build-info.outputs.skip-providers-tests != 'true' && - needs.build-info.outputs.latest-versions-only != 'true' + needs.build-info.outputs.latest-versions-only != 'true' && + needs.build-info.outputs.run-tests == 'true' with: runners: ${{ needs.build-info.outputs.amd-runners }} platform: "linux/amd64" @@ -789,10 +790,9 @@ jobs: - additional-prod-image-tests - basic-tests - build-info - - basic-tests - - generate-constraints - build-prod-images - ci-image-checks + - generate-constraints - providers - tests-helm - tests-integration-system @@ -803,7 +803,7 @@ jobs: - tests-non-db-providers - tests-postgres-core - tests-postgres-providers - - tests-special + # - tests-special - tests-sqlite-core - tests-sqlite-providers - tests-task-sdk @@ -829,8 +829,6 @@ jobs: notify-slack-failure: name: "Notify Slack on Failure" - needs: - - finalize-tests if: github.event_name == 'schedule' && failure() && github.run_attempt == 1 runs-on: ["ubuntu-22.04"] steps: @@ -843,12 +841,12 @@ jobs: # yamllint disable rule:line-length payload: | channel: "internal-airflow-ci-cd" - text: "🚨🕒 Scheduled CI Failure Alert (AMD) 🕒🚨\n\n*Details:* " + text: "🚨🕒 Scheduled CI Failure Alert (AMD) on branch *${{ github.ref_name }}* 🕒🚨\n\n*Details:* " blocks: - type: "section" text: type: "mrkdwn" - text: "🚨🕒 Scheduled CI Failure Alert (AMD) 🕒🚨\n\n*Details:* " + text: "🚨🕒 Scheduled CI Failure Alert (AMD) on branch *${{ github.ref_name }}* 🕒🚨\n\n*Details:* " # yamllint enable rule:line-length summarize-warnings: @@ -864,8 +862,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 5b8cee0a432ca..c1b39b7d3dfc0 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -130,8 +130,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: Fetch incoming commit ${{ github.sha }} with its parent uses: actions/checkout@v4 with: @@ -549,8 +549,6 @@ jobs: notify-slack-failure: name: "Notify Slack on Failure" - needs: - - finalize-tests if: github.event_name == 'schedule' && failure() && github.run_attempt == 1 runs-on: ["ubuntu-22.04"] steps: @@ -563,10 +561,10 @@ jobs: # yamllint disable rule:line-length payload: | channel: "internal-airflow-ci-cd" - text: "🚨🕒 Scheduled CI Failure Alert (ARM) 🕒🚨\n\n*Details:* " + text: "🚨🕒 Scheduled CI Failure Alert (AMD) on branch *${{ github.ref_name }}* 🕒🚨\n\n*Details:* " blocks: - type: "section" text: type: "mrkdwn" - text: "🚨🕒 Scheduled CI Failure Alert (ARM) 🕒🚨\n\n*Details:* " + text: "🚨🕒 Scheduled CI Failure Alert (AMD) on branch *${{ github.ref_name }}* 🕒🚨\n\n*Details:* " # yamllint enable rule:line-length diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index d531ffebebf2d..b52fdb906fa2e 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -120,8 +120,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 8c80213c255f5..a8535acfccbd5 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -311,7 +311,7 @@ jobs: id-token: write contents: read needs: build-docs - runs-on: ${{ fromJSON(inputs.runs-on-as-json-docs-build) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -331,27 +331,25 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ inputs.platform }} + python: ${{ inputs.default-python-version }} + use-uv: ${{ inputs.use-uv }} - name: "Download docs prepared as artifacts" uses: actions/download-artifact@v4 with: name: airflow-docs path: './generated/_build' - name: Check disk space available - run: df -h + run: df -H - name: Create /mnt/airflow-site directory run: sudo mkdir -p /mnt/airflow-site && sudo chown -R "${USER}" /mnt/airflow-site - name: "Clone airflow-site" run: > git clone https://github.com/apache/airflow-site.git /mnt/airflow-site/airflow-site && echo "AIRFLOW_SITE_DIRECTORY=/mnt/airflow-site/airflow-site" >> "$GITHUB_ENV" - - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" - uses: ./.github/actions/prepare_breeze_and_image - with: - platform: ${{ inputs.platform }} - python: ${{ inputs.default-python-version }} - use-uv: ${{ inputs.use-uv }} - name: "Publish docs" env: DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }} @@ -359,7 +357,7 @@ jobs: breeze release-management publish-docs --override-versioned --run-in-parallel ${DOCS_LIST_AS_STRING} - name: Check disk space available - run: df -h + run: df -H - name: "Generate back references for providers" run: breeze release-management add-back-references all-providers - name: "Generate back references for apache-airflow" @@ -409,8 +407,6 @@ jobs: with: fetch-depth: 2 persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 082744fd6faea..6afcbd0812219 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -103,8 +103,8 @@ jobs: with: # Needed to perform push action persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} @@ -143,7 +143,7 @@ jobs: packages: write with: runners: ${{ inputs.runners }} - platform: ${ { inputs.platform }} + platform: ${{ inputs.platform }} cache-type: "Regular AMD" include-prod-images: "true" push-latest-images: "true" diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 363888ff29718..35b31db98e575 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -67,8 +67,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh shell: bash - name: "Install Breeze" uses: ./.github/actions/breeze diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index e00ac14506ab4..26bf28cce1beb 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -96,8 +96,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 721158f097856..c5cf5cd5cef96 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -127,8 +127,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh if: inputs.upload-package-artifact == 'true' - name: "Cleanup dist and context file" shell: bash @@ -214,8 +214,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -267,6 +267,7 @@ jobs: INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} INCLUDE_NOT_READY_PROVIDERS: "true" + USE_UV: ${{ inputs.use-uv }} - name: "Verify PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" run: breeze prod-image verify - name: "Export PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 4f6ce6c762964..c72bef7003bba 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -40,10 +40,6 @@ on: # yamllint disable-line rule:truthy description: "Branch used to run the CI jobs in (main/v*_*_test)." required: true type: string - use-uv: - description: "Whether to use uv to build the image (true/false)" - required: true - type: string upgrade-to-newer-dependencies: description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)" required: true @@ -77,7 +73,7 @@ jobs: branch: ${{ inputs.branch }} # Always build images during the extra checks and never push them push-image: "false" - use-uv: ${{ inputs.use-uv }} + use-uv: "true" upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index 23e644a286de6..748a466cf8ce6 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -24,21 +24,29 @@ on: # yamllint disable-line rule:truthy description: "The branch or tag to checkout for the docs publishing" required: true type: string + include-docs: + description: | + Space separated list of docs to build. + To publish docs for nested provider packages, provide the package name with `.` + eg: amazon common.messaging apache.kafka + + required: false + default: "all-providers" + type: string exclude-docs: description: "Comma separated list of docs to exclude" required: false - default: "NO_DOCS" + default: "no-docs-excluded" type: string destination-location: - description: "The destination location in S3" - required: false - default: "s3://staging-docs-airflow-apache-org/docs" - type: string - docs-list-as-string: - description: "Space separated list of docs to build" + description: "The destination location in S3, default is live site" required: false - default: "" - type: string + default: "s3://live-docs-airflow-apache-org/docs" + type: choice + options: + - s3://live-docs-airflow-apache-org/docs + - s3://staging-docs-airflow-apache-org/docs + env: AIRFLOW_ROOT_PATH: "/home/runner/work/temp-airflow-repo-reference" # checkout dir for referenced tag permissions: @@ -48,15 +56,15 @@ jobs: timeout-minutes: 10 name: "Build Info" runs-on: ["ubuntu-24.04"] - outputs: - runners: ${{ steps.selective-checks.outputs.amd-runners }} env: GITHUB_CONTEXT: ${{ toJson(github) }} VERBOSE: true REF: ${{ inputs.ref }} + INCLUDE_DOCS: ${{ inputs.include-docs }} EXCLUDE_DOCS: ${{ inputs.exclude-docs }} DESTINATION_LOCATION: ${{ inputs.destination-location }} - DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }} + outputs: + include-docs: ${{ inputs.include-docs == 'all' && '' || inputs.include-docs }} if: contains(fromJSON('[ "ashb", "eladkal", @@ -74,29 +82,9 @@ jobs: echo "Input parameters summary" echo "=========================" echo "Ref: '${REF}'" + echo "Included docs : '${INCLUDE_DOCS}'" echo "Exclude docs: '${EXCLUDE_DOCS}'" echo "Destination location: '${DESTINATION_LOCATION}'" - echo "Docs list as string: '${DOCS_LIST_AS_STRING}'" - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: "Install uv" - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - use-uv: "true" - - name: Selective checks - id: selective-checks - env: - VERBOSE: "false" - run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} build-ci-images: name: Build CI images @@ -108,7 +96,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runners: ${{ needs.build-info.outputs.amd-runners }} + runners: '["ubuntu-22.04"]' platform: "linux/amd64" push-image: "false" upload-image-artifact: "true" @@ -122,7 +110,7 @@ jobs: disable-airflow-repo-cache: false build-docs: - needs: [build-ci-images] + needs: [build-ci-images, build-info] timeout-minutes: 150 name: "Build documentation" runs-on: ubuntu-latest @@ -132,7 +120,7 @@ jobs: GITHUB_USERNAME: ${{ github.actor }} INCLUDE_NOT_READY_PROVIDERS: "true" INCLUDE_SUCCESS_OUTPUTS: false - PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }} + PYTHON_MAJOR_MINOR_VERSION: 3.9 VERBOSE: "true" steps: - name: "Cleanup repo" @@ -156,9 +144,9 @@ jobs: use-uv: true - name: "Building docs with --docs-only flag" env: - DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }} + INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }} run: > - breeze build-docs ${DOCS_LIST_AS_STRING} --docs-only + breeze build-docs ${INCLUDE_DOCS} --docs-only --include-commits - name: "Upload build docs" uses: actions/upload-artifact@v4 with: @@ -169,7 +157,7 @@ jobs: overwrite: 'true' publish-docs-to-s3: - needs: [build-docs] + needs: [build-docs, build-info] name: "Publish documentation to S3" permissions: id-token: write @@ -191,8 +179,12 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare breeze & CI image: 3.9" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: "linux/amd64" + python: 3.9 + use-uv: true - name: "Checkout ${{ inputs.ref }}" run: | git clone https://github.com/apache/airflow.git "${AIRFLOW_ROOT_PATH}" @@ -205,27 +197,21 @@ jobs: name: airflow-docs path: ${{ env.AIRFLOW_ROOT_PATH }}/generated/_build - name: Check disk space available - run: df -h + run: df -H # Here we will create temp airflow-site dir to publish # docs and for back-references - name: Create /mnt/airflow-site directory run: | sudo mkdir -p /mnt/airflow-site && sudo chown -R "${USER}" /mnt/airflow-site echo "AIRFLOW_SITE_DIRECTORY=/mnt/airflow-site/" >> "$GITHUB_ENV" - - name: "Prepare breeze & CI image: 3.9" - uses: ./.github/actions/prepare_breeze_and_image - with: - platform: "linux/amd64" - python: 3.9 - use-uv: true - name: "Publish docs to tmp directory" env: - DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }} + INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }} run: > breeze release-management publish-docs --override-versioned --run-in-parallel - ${DOCS_LIST_AS_STRING} + ${INCLUDE_DOCS} - name: Check disk space available - run: df -h + run: df -H - name: "Generate back references for providers" run: breeze release-management add-back-references all-providers - name: "Generate back references for apache-airflow" @@ -255,4 +241,4 @@ jobs: run: | breeze release-management publish-docs-to-s3 --source-dir-path ${SOURCE_DIR_PATH} \ --destination-location ${DESTINATION_LOCATION} --stable-versions \ - --exclude-docs ${EXCLUDE_DOCS} + --exclude-docs ${EXCLUDE_DOCS} --overwrite diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index d562f798c7791..f2258c13b77a9 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -116,8 +116,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -187,8 +187,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 9d8655ecaead3..fe1ce300fbf4f 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -25,8 +25,12 @@ on: # yamllint disable-line rule:truthy required: true amdOnly: type: boolean - description: 'Limit to amd64 only (faster testing)' + description: 'Limit to amd64 images' default: false + limitPythonVersions: + type: string + description: 'Force python versions (e.g. "3.9 3.10")' + default: '' permissions: contents: read packages: read @@ -42,12 +46,10 @@ jobs: name: "Build Info" runs-on: ["ubuntu-24.04"] outputs: - pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} - allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} - defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} + pythonVersions: ${{ steps.determine-python-versions.outputs.python-versions }} platformMatrix: ${{ steps.determine-matrix.outputs.platformMatrix }} airflowVersion: ${{ steps.check-airflow-version.outputs.airflowVersion }} - skipLatest: ${{ steps.selective-checks.outputs.skipLatest }} + skipLatest: ${{ steps.check-airflow-version.outputs.skip-latest }} amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} env: @@ -55,6 +57,7 @@ jobs: VERBOSE: true AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} AMD_ONLY: ${{ github.event.inputs.amdOnly }} + LIMIT_PYTHON_VERSIONS: ${{ github.event.inputs.limitPythonVersions }} if: contains(fromJSON('[ "ashb", "eladkal", @@ -73,15 +76,18 @@ jobs: echo "=========================" echo "Airflow version: '${AIRFLOW_VERSION}'" echo "AMD only: '${AMD_ONLY}'" + echo "Limit python versions: '${LIMIT_PYTHON_VERSIONS}'" - name: "Cleanup repo" shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + run: > + docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install uv" run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: "Check airflow version" @@ -106,195 +112,39 @@ jobs: else echo 'platformMatrix=["linux/amd64", "linux/arm64"]' >> "${GITHUB_OUTPUT}" fi - - build-images: - timeout-minutes: 50 - # yamllint disable rule:line-length - name: "Build: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }}" - runs-on: ${{ (matrix.platform == 'linux/amd64') && fromJSON(needs.build-info.outputs.amd-runners) || fromJSON(needs.build-info.outputs.arm-runners) }} - needs: [build-info] - strategy: - fail-fast: false - matrix: - python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} - platform: ${{ fromJSON(needs.build-info.outputs.platformMatrix) }} - env: - AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - PLATFORM: ${{ matrix.platform }} - SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' && '--skip-latest' || '' }} - COMMIT_SHA: ${{ github.sha }} - REPOSITORY: ${{ github.repository }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - use-uv: "false" - - name: Free space - run: breeze ci free-space --answer yes - - name: "Cleanup dist and context file" - run: rm -fv ./dist/* ./docker-context-files/* - - name: "Login to hub.docker.com" - run: > - echo ${{ secrets.DOCKERHUB_TOKEN }} | - docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} - - name: "Get env vars for metadata" + - name: "Determine python versions" shell: bash - run: | - echo "ARTIFACT_NAME=metadata-${PYTHON_MAJOR_MINOR_VERSION}-${PLATFORM/\//_}" >> "${GITHUB_ENV}" - echo "MANIFEST_FILE_NAME=metadata-${AIRFLOW_VERSION}-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" - echo "MANIFEST_SLIM_FILE_NAME=metadata-${AIRFLOW_VERSION}-slim-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" - - name: Login to ghcr.io + id: determine-python-versions env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ACTOR: ${{ github.actor }} - run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin - - name: "Install buildx plugin" + ALL_PYTHON_VERSIONS: ${{ steps.selective-checks.outputs.all-python-versions }} # yamllint disable rule:line-length run: | - sudo apt-get update - sudo apt-get install ca-certificates curl - sudo install -m 0755 -d /etc/apt/keyrings - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc + # override python versions if specified + if [[ "${LIMIT_PYTHON_VERSIONS}" != "" ]]; then + PYTHON_VERSIONS=$(python3 -c "import json; print(json.dumps('${LIMIT_PYTHON_VERSIONS}'.split(' ')))") + else + PYTHON_VERSIONS=${ALL_PYTHON_VERSIONS} + fi + echo "python-versions=${PYTHON_VERSIONS}" >> "${GITHUB_OUTPUT}" - # Add the repository to Apt sources: - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update - sudo apt install docker-buildx-plugin - - name: "Create airflow_cache builder" - run: docker buildx create --name airflow_cache --driver docker-container - - name: > - Build regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} - run: > - breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} - --python ${PYTHON_MAJOR_MINOR_VERSION} - --metadata-folder dist - - name: > - Verify regular image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} - run: > - breeze prod-image verify --pull --manifest-file dist/${MANIFEST_FILE_NAME} - - name: > - Release slim images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} - run: > - breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} - --python ${PYTHON_MAJOR_MINOR_VERSION} --slim-images - --metadata-folder dist - - name: > - Verify slim image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} - run: > - breeze prod-image verify --pull --slim-image --manifest-file dist/${MANIFEST_SLIM_FILE_NAME} - - name: "List upload-able artifacts" - shell: bash - run: find ./dist -name '*.json' - - name: "Upload metadata artifact ${{ env.ARTIFACT_NAME }}" - uses: actions/upload-artifact@v4 - with: - name: ${{ env.ARTIFACT_NAME }} - path: ./dist/metadata-* - retention-days: 7 - if-no-files-found: error - - name: "Docker logout" - run: docker logout - if: always() - merge-images: - timeout-minutes: 5 - name: "Merge: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" - runs-on: ["ubuntu-22.04"] - needs: [build-info, build-images] + release-images: + name: "Release images" + needs: [build-info] strategy: fail-fast: false matrix: - python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} - env: - AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' && '--skip-latest' || '' }} - COMMIT_SHA: ${{ github.sha }} - REPOSITORY: ${{ github.repository }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - use-uv: "false" - - name: Free space - run: breeze ci free-space --answer yes - - name: "Cleanup dist and context file" - run: rm -fv ./dist/* ./docker-context-files/* - - name: "Login to hub.docker.com" - run: > - echo ${{ secrets.DOCKERHUB_TOKEN }} | - docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} - - name: Login to ghcr.io - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ACTOR: ${{ github.actor }} - run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin - - name: "Download metadata artifacts" - uses: actions/download-artifact@v4 - with: - path: ./dist - pattern: metadata-${{ matrix.python-version }}-* - - name: "List downloaded artifacts" - shell: bash - run: find ./dist -name '*.json' - - name: "Install buildx plugin" - # yamllint disable rule:line-length - run: | - sudo apt-get update - sudo apt-get install ca-certificates curl - sudo install -m 0755 -d /etc/apt/keyrings - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc - - # Add the repository to Apt sources: - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update - sudo apt install docker-buildx-plugin - - name: "Install regctl" - # yamllint disable rule:line-length - run: | - mkdir -p ~/bin - curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl - chmod 755 ${HOME}/bin/regctl - echo "${HOME}/bin" >>${GITHUB_PATH} - - name: "Merge regular images ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" - run: > - breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} - --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist - - name: "Merge slim images ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" - run: > - breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} - --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist --slim-images - - name: "Docker logout" - run: docker logout - if: always() + python: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} + uses: ./.github/workflows/release_single_dockerhub_image.yml + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + permissions: + contents: read + with: + pythonVersion: ${{ matrix.python }} + airflowVersion: ${{ needs.build-info.outputs.airflowVersion }} + platformMatrix: ${{ needs.build-info.outputs.platformMatrix }} + skipLatest: ${{ needs.build-info.outputs.skipLatest }} + armRunners: ${{ needs.build-info.outputs.arm-runners }} + amdRunners: ${{ needs.build-info.outputs.amd-runners }} diff --git a/.github/workflows/release_single_dockerhub_image.yml b/.github/workflows/release_single_dockerhub_image.yml new file mode 100644 index 0000000000000..55a8c2cc00429 --- /dev/null +++ b/.github/workflows/release_single_dockerhub_image.yml @@ -0,0 +1,242 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: "Release single PROD image" +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + airflowVersion: + description: 'Airflow version (e.g. 3.0.1, 3.0.1rc1, 3.0.1b1)' + type: string + required: true + platformMatrix: + description: 'Platform matrix formatted as json (e.g. ["linux/amd64", "linux/arm64"])' + type: string + required: true + pythonVersion: + description: 'Python version (e.g. 3.8, 3.9, 3.10, 3.11)' + type: string + required: true + skipLatest: + description: "Skip tagging latest release (true/false)" + type: string + required: true + amdRunners: + description: "Amd64 runners (e.g. [\"ubuntu-22.04\", \"ubuntu-24.04\"])" + type: string + required: true + armRunners: + description: "Arm64 runners (e.g. [\"ubuntu-22.04\", \"ubuntu-24.04\"])" + type: string + required: true + secrets: + DOCKERHUB_USER: + required: true + DOCKERHUB_TOKEN: + required: true +permissions: + contents: read +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERBOSE: true +jobs: + build-images: + timeout-minutes: 50 + # yamllint disable rule:line-length + name: "Build: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }}" + runs-on: ${{ (matrix.platform == 'linux/amd64') && fromJSON(inputs.amdRunners) || fromJSON(inputs.armRunners) }} + strategy: + fail-fast: false + matrix: + platform: ${{ fromJSON(inputs.platformMatrix) }} + env: + AIRFLOW_VERSION: ${{ inputs.airflowVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.pythonVersion }} + PLATFORM: ${{ matrix.platform }} + SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }} + COMMIT_SHA: ${{ github.sha }} + REPOSITORY: ${{ github.repository }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: "false" + - name: Free space + run: breeze ci free-space --answer yes + - name: "Cleanup dist and context file" + run: rm -fv ./dist/* ./docker-context-files/* + - name: "Login to hub.docker.com" + run: > + echo ${{ secrets.DOCKERHUB_TOKEN }} | + docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} + - name: "Get env vars for metadata" + shell: bash + run: | + echo "ARTIFACT_NAME=metadata-${PYTHON_MAJOR_MINOR_VERSION}-${PLATFORM/\//_}" >> "${GITHUB_ENV}" + echo "MANIFEST_FILE_NAME=metadata-${AIRFLOW_VERSION}-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" + echo "MANIFEST_SLIM_FILE_NAME=metadata-${AIRFLOW_VERSION}-slim-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" + - name: Login to ghcr.io + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACTOR: ${{ github.actor }} + run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin + - name: "Install buildx plugin" + # yamllint disable rule:line-length + run: | + sudo apt-get update + sudo apt-get install ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt install docker-buildx-plugin + - name: "Create airflow_cache builder" + run: docker buildx create --name airflow_cache --driver docker-container + - name: > + Build regular images: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} + run: > + breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} + --metadata-folder dist + - name: > + Verify regular image: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} + run: > + breeze prod-image verify --pull --manifest-file dist/${MANIFEST_FILE_NAME} + - name: > + Release slim images: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} + run: > + breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} --slim-images + --metadata-folder dist + - name: > + Verify slim image: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} + run: > + breeze prod-image verify --pull --slim-image --manifest-file dist/${MANIFEST_SLIM_FILE_NAME} + - name: "List upload-able artifacts" + shell: bash + run: find ./dist -name '*.json' + - name: "Upload metadata artifact ${{ env.ARTIFACT_NAME }}" + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: ./dist/metadata-* + retention-days: 7 + if-no-files-found: error + - name: "Docker logout" + run: docker logout + if: always() + + merge-images: + timeout-minutes: 5 + name: "Merge: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}" + runs-on: ["ubuntu-22.04"] + needs: [build-images] + env: + AIRFLOW_VERSION: ${{ inputs.airflowVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.pythonVersion }} + SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }} + COMMIT_SHA: ${{ github.sha }} + REPOSITORY: ${{ github.repository }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Prepare and cleanup runner" + run: ./scripts/ci/prepare_and_cleanup_runner.sh + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: "false" + - name: Free space + run: breeze ci free-space --answer yes + - name: "Cleanup dist and context file" + run: rm -fv ./dist/* ./docker-context-files/* + - name: "Login to hub.docker.com" + run: > + echo ${{ secrets.DOCKERHUB_TOKEN }} | + docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} + - name: Login to ghcr.io + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACTOR: ${{ github.actor }} + run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin + - name: "Download metadata artifacts" + uses: actions/download-artifact@v4 + with: + path: ./dist + pattern: metadata-${{ inputs.pythonVersion }}-* + - name: "List downloaded artifacts" + shell: bash + run: find ./dist -name '*.json' + - name: "Install buildx plugin" + # yamllint disable rule:line-length + run: | + sudo apt-get update + sudo apt-get install ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt install docker-buildx-plugin + - name: "Install regctl" + # yamllint disable rule:line-length + run: | + mkdir -p ~/bin + curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl + chmod 755 ${HOME}/bin/regctl + echo "${HOME}/bin" >>${GITHUB_PATH} + - name: "Merge regular images ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}" + run: > + breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist + - name: "Merge slim images ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}" + run: > + breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist --slim-images + - name: "Docker logout" + run: docker logout + if: always() diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 435b49897da14..4b99ac6031137 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -124,7 +124,7 @@ permissions: contents: read jobs: tests: - timeout-minutes: 120 + timeout-minutes: 65 name: "\ ${{ inputs.test-scope }}-${{ inputs.test-group }}:\ ${{ inputs.test-name }}${{ inputs.test-name-separator }}${{ matrix.backend-version }}:\ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1fc3e81f2ad48..01a585c19c456 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ default_stages: [pre-commit, pre-push] default_language_version: python: python3 - node: 22.15.0 + node: 22.16.0 minimum_pre_commit_version: '3.2.0' exclude: ^.*/.*_vendor/ repos: @@ -35,7 +35,15 @@ repos: - id: doctoc name: Add TOC for Markdown and RST files files: - ^README\.md$|^UPDATING.*\.md$|^chart/UPDATING.*\.md$|^dev/.*\.md$|^dev/.*\.rst$|^\.github/.*\.md|^airflow-core/tests/system/README\.md$ + (?x) + ^README\.md$| + ^UPDATING.*\.md$| + ^chart/UPDATING.*\.md$| + ^dev/.*\.md$| + ^dev/.*\.rst$| + ^docs/README\.md$| + ^\.github/.*\.md$| + ^airflow-core/tests/system/README\.md$ args: - "--maxlevel" - "2" @@ -96,7 +104,7 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all Python files - exclude: ^\.github/.*$|^.*/_vendor/.*$ + exclude: ^\.github/.*$|^.*/_vendor/.*$|^airflow-ctl/.*/.*generated\.py$ files: \.py$|\.pyi$ args: - --comment-style @@ -125,7 +133,14 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all YAML files except Helm templates - exclude: ^\.github/.*$|^chart/templates/.*|.*reproducible_build\.yaml$|^.*/v1.*\.yaml$|^.*/openapi/_private_ui.*\.yaml$|^.*/pnpm-lock\.yaml$ + exclude: > + (?x) + ^\.github/.*$|^chart/templates/.*| + .*reproducible_build\.yaml$| + ^.*/v2.*\.yaml$| + ^.*/openapi/_private_ui.*\.yaml$| + ^.*/pnpm-lock\.yaml$| + .*-generated\.yaml$ types: [yaml] files: \.ya?ml$ args: @@ -210,7 +225,8 @@ repos: language: python entry: ./scripts/ci/pre_commit/check_deferrable_default.py pass_filenames: false - additional_dependencies: ['libcst>=1.1.0'] + # libcst doesn't have source wheels for all PY except PY3.12, excluding it + additional_dependencies: ['libcst>=1.1.0,!=1.8.0'] files: ^(providers/.*/)?airflow/.*/(sensors|operators)/.*\.py$ - repo: https://github.com/asottile/blacken-docs rev: 1.19.1 @@ -246,6 +262,9 @@ repos: name: Check that executables have shebang - id: check-xml name: Check XML files with xmllint + exclude: > + (?x) + ^scripts/ci/docker-compose/gremlin/. - id: trailing-whitespace name: Remove trailing whitespace at end of line exclude: ^airflow-core/docs/img/.*\.dot$|^dev/breeze/doc/images/output.*$ @@ -270,7 +289,7 @@ repos: - id: python-no-log-warn name: Check if there are no deprecate log warn - repo: https://github.com/adrienverge/yamllint - rev: v1.36.2 + rev: v1.37.1 hooks: - id: yamllint name: Check YAML files with yamllint @@ -311,10 +330,10 @@ repos: exclude: material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$|^.*/kinglear\.txt$|^.*pnpm-lock\.yaml$|.*/dist/.* args: - --ignore-words=docs/spelling_wordlist.txt - - --skip=providers/.*/src/airflow/providers/*/*.rst,providers/*/docs/changelog.rst,docs/*/commits.rst,providers/*/docs/commits.rst,providers/*/*/docs/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md + - --skip=providers/.*/src/airflow/providers/*/*.rst,providers/*/docs/changelog.rst,docs/*/commits.rst,providers/*/docs/commits.rst,providers/*/*/docs/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md,*.svg - --exclude-file=.codespellignorelines - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.5.1 + rev: v1.7.0 hooks: - id: zizmor name: Run zizmor to check for github workflow syntax errors @@ -476,7 +495,10 @@ repos: name: Generate airflow diagrams entry: ./scripts/ci/pre_commit/generate_airflow_diagrams.py language: python - files: ^airflow-core/docs/.*/diagram_[^/]*\.py$ + files: > + (?x) + ^airflow-core/docs/.*/diagram_[^/]*\.py$| + ^docs/images/.*\.py$ pass_filenames: true additional_dependencies: ['rich>=12.4.4', 'diagrams>=0.23.4'] - id: generate-volumes-for-sources @@ -583,7 +605,6 @@ repos: pass_filenames: true exclude: > (?x) - ^clients/python/openapi_v1.yaml$| ^airflow-core/src/airflow/ui/openapi-gen/| ^airflow-core/src/airflow/cli/commands/local_commands/fastapi_api_command\.py$| ^airflow-core/src/airflow/config_templates/| @@ -645,7 +666,7 @@ repos: ^.*commits\.(rst|txt)$| ^.*RELEASE_NOTES\.rst$| ^contributing-docs/03_contributors_quick_start\.rst$| - ^.*\.(png|gif|jp[e]?g|tgz|lock)$| + ^.*\.(png|gif|jp[e]?g|svg|tgz|lock)$| git| ^airflow-core/newsfragments/43349\.significant\.rst$| ^airflow-core/newsfragments/41368\.significant\.rst$| @@ -970,6 +991,7 @@ repos: exclude: > (?x) ^scripts/ci/docker-compose/grafana/.| + ^scripts/ci/docker-compose/gremlin/.| ^scripts/ci/docker-compose/.+-config\.ya?ml$ require_serial: true additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] @@ -1295,6 +1317,7 @@ repos: language: python entry: ./scripts/ci/pre_commit/mypy.py files: ^airflow-ctl/src/airflowctl/.*\.py$|^airflow-ctl/tests/.*\.py$ + exclude: .*generated.py require_serial: true additional_dependencies: ['rich>=12.4.4'] - id: mypy-airflow-ctl @@ -1309,11 +1332,19 @@ repos: - id: generate-openapi-spec name: Generate the FastAPI API spec language: python - entry: ./scripts/ci/pre_commit/update_fastapi_api_spec.py + entry: ./scripts/ci/pre_commit/generate_openapi_spec.py pass_filenames: false files: ^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$|^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ exclude: ^airflow-core/src/airflow/api_fastapi/execution_api/.* - additional_dependencies: ['rich>=12.4.4'] + additional_dependencies: ['rich>=12.4.4', 'openapi-spec-validator>=0.7.1'] + - id: generate-openapi-spec-fab + name: Generate the FastAPI API spec for FAB + language: python + entry: ./scripts/ci/pre_commit/generate_openapi_spec_fab.py + pass_filenames: false + files: ^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$|^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ + exclude: ^airflow-core/src/airflow/api_fastapi/execution_api/.* + additional_dependencies: ['rich>=12.4.4', 'openapi-spec-validator>=0.7.1'] - id: ts-compile-format-lint-ui name: Compile / format / lint UI description: TS types generation / ESLint / Prettier new UI files diff --git a/.rat-excludes b/.rat-excludes index 75a85f9873fa1..95e31f35fba37 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -161,9 +161,17 @@ PKG-INFO .openapi-generator-ignore version.txt v1*.yaml +v2*.yaml _private_ui*.yaml # Front end generated files api-generated.ts openapi-gen pnpm-lock.yaml + +# python generated file +generated.py +auth_generated.py + +# hash files +www-hash.txt diff --git a/Dockerfile b/Dockerfile index f705e48d4f029..0ca2986087de6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,8 +56,8 @@ ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` ARG AIRFLOW_PIP_VERSION=25.1.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_SETUPTOOLS_VERSION=80.1.0 -ARG AIRFLOW_UV_VERSION=0.7.2 +ARG AIRFLOW_SETUPTOOLS_VERSION=80.8.0 +ARG AIRFLOW_UV_VERSION=0.7.8 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" @@ -455,7 +455,12 @@ function common::get_packaging_tool() { echo export PACKAGING_TOOL="uv" export PACKAGING_TOOL_CMD="uv pip" - export EXTRA_INSTALL_FLAGS="--group=dev" + if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then + # for uv only install dev group when we install from sources + export EXTRA_INSTALL_FLAGS="--group=dev" + else + export EXTRA_INSTALL_FLAGS="" + fi export EXTRA_UNINSTALL_FLAGS="" export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --resolution highest" export UPGRADE_IF_NEEDED="--upgrade" diff --git a/Dockerfile.ci b/Dockerfile.ci index 39a5373ee1c83..4c46efbf2dc2c 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -394,7 +394,12 @@ function common::get_packaging_tool() { echo export PACKAGING_TOOL="uv" export PACKAGING_TOOL_CMD="uv pip" - export EXTRA_INSTALL_FLAGS="--group=dev" + if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then + # for uv only install dev group when we install from sources + export EXTRA_INSTALL_FLAGS="--group=dev" + else + export EXTRA_INSTALL_FLAGS="" + fi export EXTRA_UNINSTALL_FLAGS="" export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --resolution highest" export UPGRADE_IF_NEEDED="--upgrade" @@ -1354,8 +1359,8 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` ARG AIRFLOW_PIP_VERSION=25.1.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_SETUPTOOLS_VERSION=80.1.0 -ARG AIRFLOW_UV_VERSION=0.7.2 +ARG AIRFLOW_SETUPTOOLS_VERSION=80.8.0 +ARG AIRFLOW_UV_VERSION=0.7.8 # TODO(potiuk): automate with upgrade check (possibly) ARG AIRFLOW_PRE_COMMIT_VERSION="4.2.0" ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4" diff --git a/INSTALLING.md b/INSTALLING.md new file mode 100644 index 0000000000000..e62b66a7a2a0e --- /dev/null +++ b/INSTALLING.md @@ -0,0 +1,103 @@ + + +## Local Development Setup + +This section outlines a recommended approach for setting up a local development environment for Apache Airflow on macOS and Linux, primarily using PyEnv for Python version management. + +> ⚠️ Avoid using either system-installed Python or Python from Homebrew, as these versions are often labeled `--externally-managed` resulting in restricted dependency installation. + +You can use other ways to install Python and airflow. Airflow development setup requires `uv` and if you want to setup environment for development, `uv` is the only supported local development environment setup, because we are using `uv workspace` extensively. See [local virtualenv setup in contributing docs](https://github.com/apache/airflow/blob/main/contributing-docs/07_local_virtualenv.rst) for details. + +If you are just installing airflow to run it locally, You can use other ways to set up your Python and virtualenv: `uv` is one of the options (refer to `uv` documentation), but you can also use more traditional tools - for example `pyenv`. Note that it is recommended to install airflow with constraints - at least initially - because this way you can reproducibly install airflow. See [Installation from PyPI](https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html) for more details. + +### ✅ Setup using pyenv: + +1. **Install pyenv (macOS and Linux)**: + +```bash +brew install pyenv +``` + +(Note: Homebrew is the recommended method on macOS. For Linux, you can typically install pyenv using the `pyenv-installer` script as detailed in the official documentation: [https://github.com/pyenv/pyenv#installation](https://github.com/pyenv/pyenv#installation).) + +2. **Install Python**: + +```bash +pyenv install 3.11.9 +pyenv global 3.11.9 +``` + +3. **Check Python version**: + +```bash +python --version +``` + +4. **Create and Activate a Virtual Environment**: Since Apache Airflow requires multiple dependencies, it's a good practice to isolate these dependencies in a virtual environment. + +- Create a virtual environment: + +```bash +python -m venv airflow_venv +``` + +- Activate the virtual environment: + +```bash +source airflow_venv/bin/activate +``` + +5. **Install Apache Airflow**: Apache Airflow is available on PyPI. To install it, you can use the following command in your terminal: + +```bash +pip install apache-airflow==3.0.0 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.0.0/constraints-3.11.txt" +``` + +Note that installing with constraints - at least initially - is recommended for reproducible installation. It might sometimes happen that 3rd-party distributions are released and their latest versions break airflow. Using constraints makes the installation reproducible with versions of dependencies that were "frozen" at the time of releasing airflow. Note you have to specify both - airflow version and Python version you are using. + +You can also specify additional extras - when you want to install airflow with additional providers: + +```bash +pip install apache-airflow[amazon,google]==3.0.0 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.0.0/constraints-3.11.txt" +``` + +6. **Set the AIRFLOW_HOME Environment Variable**: Apache Airflow requires a directory to store configuration files, logs, and other data. Set the AIRFLOW_HOME variable to specify this directory. + +- Set the Airflow home directory: + +```bash +export AIRFLOW_HOME=~/airflow +``` + +7. **Run Airflow in standalone mode**: Apache Airflow runs several components, like the scheduler, web server, and API server, to manage workflows and show the UI. + +- To run Airflow in standalone mode (which will automatically start the required components): + +```bash +airflow standalone +``` + +8. **Access the Airflow Web UI**: Once the components are up and running, you can access the Airflow UI through your browser: + +- Open your browser and go to: + +```text +http://localhost:8080 +``` diff --git a/README.md b/README.md index a0360260691e6..4048392413c15 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,9 @@ # Apache Airflow [![PyPI version](https://badge.fury.io/py/apache-airflow.svg)](https://badge.fury.io/py/apache-airflow) -[![GitHub Build](https://github.com/apache/airflow/actions/workflows/ci.yml/badge.svg)](https://github.com/apache/airflow/actions) +[![GitHub Build main](https://github.com/apache/airflow/actions/workflows/ci-amd.yml/badge.svg)](https://github.com/apache/airflow/actions) +[![GitHub Build 3.0](https://github.com/apache/airflow/actions/workflows/ci-amd.yml/badge.svg?branch=v3-0-test)](https://github.com/apache/airflow/actions) +[![GitHub Build 2.11](https://github.com/apache/airflow/actions/workflows/ci.yml/badge.svg?branch=v2-11-test)](https://github.com/apache/airflow/actions) [![Coverage Status](https://codecov.io/gh/apache/airflow/graph/badge.svg?token=WdLKlKHOAU)](https://codecov.io/gh/apache/airflow) [![License](https://img.shields.io/:license-Apache%202-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0.txt) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/apache-airflow.svg)](https://pypi.org/project/apache-airflow/) @@ -58,6 +60,7 @@ Use Airflow to author workflows as directed acyclic graphs (DAGs) of tasks. The - [Requirements](#requirements) - [Getting started](#getting-started) - [Installing from PyPI](#installing-from-pypi) +- [Installation](#installation) - [Official source code](#official-source-code) - [Convenience packages](#convenience-packages) - [User Interface](#user-interface) @@ -139,6 +142,7 @@ Documentation for dependent projects like provider distributions, Docker image, + ## Installing from PyPI We publish Apache Airflow as `apache-airflow` package in PyPI. Installing it however might be sometimes tricky @@ -154,7 +158,6 @@ constraints files separately per major/minor Python version. You can use them as constraint files when installing Airflow from PyPI. Note that you have to specify correct Airflow tag/version/branch and Python versions in the URL. - 1. Installing just Airflow: > Note: Only `pip` installation is currently officially supported. @@ -189,6 +192,11 @@ For information on installing provider distributions, check [providers](http://airflow.apache.org/docs/apache-airflow-providers/index.html). + +## Installation + +For comprehensive instructions on setting up your local development environment and installing Apache Airflow, please refer to the [INSTALLING.md](INSTALLING.md) file. + ## Official source code @@ -291,8 +299,8 @@ Apache Airflow version life cycle: | Version | Current Patch/Minor | State | First Release | Limited Maintenance | EOL/Terminated | |-----------|-----------------------|-----------|-----------------|-----------------------|------------------| -| 3 | 3.0.1 | Supported | Apr 22, 2025 | TBD | TBD | -| 2 | 2.10.5 | Supported | Dec 17, 2020 | TBD | TBD | +| 3 | 3.0.2 | Supported | Apr 22, 2025 | TBD | TBD | +| 2 | 2.11.0 | Supported | Dec 17, 2020 | Oct 22, 2025 | Apr 22, 2026 | | 1.10 | 1.10.15 | EOL | Aug 27, 2018 | Dec 17, 2020 | June 17, 2021 | | 1.9 | 1.9.0 | EOL | Jan 03, 2018 | Aug 27, 2018 | Aug 27, 2018 | | 1.8 | 1.8.2 | EOL | Mar 19, 2017 | Jan 03, 2018 | Jan 03, 2018 | @@ -426,7 +434,7 @@ Want to help build Apache Airflow? Check out our [contributors' guide](https://g If you can't wait to contribute, and want to get started asap, check out the [contribution quickstart](https://github.com/apache/airflow/blob/main/contributing-docs/03_contributors_quick_start.rst) here! -Official Docker (container) images for Apache Airflow are described in [images](dev/breeze/doc/ci/02_images.md). +Official Docker (container) images for Apache Airflow are described in [images](https://github.com/apache/airflow/blob/main/dev/breeze/doc/ci/02_images.md). diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index f4ff9d9323415..1c836ce7fab1a 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -19,8 +19,123 @@ :local: :depth: 1 +.. note:: + Release notes for older versions can be found in the versioned documentation. + .. towncrier release notes start +Airflow 3.0.2 (2025-06-10) +-------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +No significant changes. + +Bug Fixes +""""""""" + +- Fix memory leak in dag-processor (#50558) +- Add back invalid inlet and outlet check before running tasks (#50773) +- Implement slice on LazyXComSequence to allow filtering items from a mapped task(#50117) +- Fix execution API server URL handling for relative paths in KE (#51183) +- Add log lookup exception for Empty operator subtypes (#50325) +- Increase the max zoom on the graph view to make it easier to see small dags on big monitor screens (#50772) +- Fix timezone selection and dashboard layout (#50463) +- Creating backfill for a dag is affecting other dags (#50577) +- Fix next asset schedule and dag card UX (#50271) +- Add bundle path to ``sys.path`` in task runner (#51318) +- Add bundle path to ``sys.path`` in dag processor (#50385) +- Prevent CPU spike in task supervisor when heartbeat timeout exceeded (#51023) +- Fix Airflow Connection Form widget error (#51168) +- Add backwards compatibility shim and deprecation warning for EmailOperator (#51004) +- Handle ``SIGSEGV`` signals during DAG file imports (#51171) +- Fix deferred task resumption in ``dag.test()`` (#51182) +- Fix get dags query to not have join explosion (#50984) +- Ensure Logical date is populated correctly in Context vars (#50898) +- Mask variable values in task logs only if the variable key is sensitive (#50775) +- Mask secrets when retrieving variables from secrets backend (#50895) +- Deserialize should work while retrieving variables with secrets backend (#50889) +- Fix XCom deserialization for mapped tasks with custom backend (#50687) +- Support macros defined via plugins in Airflow 3 (#50642) +- Fix Pydantic ``ForwardRef`` error by reordering discriminated union definitions (#50688) +- Adding backwards compatibility shim for ``BaseNotifier`` (#50340) +- Use latest bundle version when clearing / re-running dag (#50040) +- Handle ``upstream_mapped_index`` when xcom access is needed (#50641) +- Remove unnecessary breaking flag in config command (#50781) +- Do not flood worker logs with secrets backend loading logs (#50581) +- Persist table sorting preferences across sessions using local storage (#50720) +- Fixed patch_task_instance API endpoint to support task instance summaries and task groups (#50550) +- Fixed bulk API schemas to improve OpenAPI compatibility and client generation (#50852) +- Fixed variable API endpoints to support keys containing slashes (#50841) +- Restored backward compatibility for the ``/run`` API endpoint for older Task SDK clients +- Fixed dropdown overflow and error text styling in ``FlexibleForm`` component (#50845) +- Corrected DAG tag rendering to display ``+1 more`` when tags exceed the display limit by one (#50669) +- Fix permission check on the ui config endpoint (#50608) +- Fix ``default_args`` handling in operator ``.partial()`` to prevent ``TypeError`` when unused keys are present (#50525) +- DAG Processor: Fix index to sort by last parsing duration (#50388) +- UI: Fix border overlap issue in the Events page (#50453) +- Fix ``airflow tasks clear`` command (#49631) +- Restored support for ``--local`` flag in ``dag list`` and ``dag list-import-errors`` CLI commands (#49380) +- CLI: Exclude example dags when a bundle is passed (#50401) +- Fix CLI export to handle stdout without file descriptors (#50328) +- Fix ``DagProcessor`` stats log to show the correct parse duration (#50316) +- Fix OpenAPI schema for ``get_log`` API (#50547) + +Miscellaneous +""""""""""""" + +- UI: Implement navigation on bar click (#50416) +- UI: Always Show Trends count in Dag Overview (#50183) +- UI: Add basic json check to variable value +- Remove filtering by last dag run state in patch dags endpoint (#51347) +- Ensure that both public and ui dags endpoints map to DagService (#51226) +- Refresh Dag details page on new run (#51173) +- Log fallback to None when no XCom value is found (#51285) +- Move ``example_dags`` in standard provider to ``example_dags`` in sources (#51275) +- Bring back "standard" example dags to the ``airflow-core`` package (#51192) +- Faster note on grid endpoint (#51247) +- Port ``task.test`` to Task SDK (#50827) +- Port ``dag.test`` to Task SDK (#50300,#50419) +- Port ``ti.run`` to Task SDK execution path (#50141) +- Support running ``airflow dags test`` from local files (#50420) +- Move macros to task SDK ``execution_time`` module (#50940) +- Add a link to the Airflow logo in Nav (#50304) +- UI: Bump minor and patch package json dependencies (#50298) +- Added a direct link to the latest DAG run in the DAG header (#51119,#51148) +- Fetch only the most recent ``dagrun`` value for list display (#50834) +- Move ``secret_key`` config to ``api`` section (#50839) +- Move various ``webserver`` configs to ``fab`` provider (#50774,#50269,#50208,#50896) +- Make ``dag_run`` nullable in Details page (#50719) +- Rename Operation IDs for task instance endpoints to include map indexes (#49608) +- Update default sort for connections and dags (#50600) +- Raise exception if downgrade can't proceed due to no ``ab_user`` table (#50343) +- Enable JSON serialization for variables created via the bulk API (#51057) +- Always display the backfill option in the UI; enable it only for DAGs with a defined schedule (#50969) +- Optimized DAG header to fetch only the most recent DAG run for improved performance (#50767) +- Add ``owner_links`` field to ``DAGDetailsResponse`` for enhanced owner metadata in the API (#50557) +- UI: Move map index column to be in line with other columns when viewing a summary mapped tasks (#50302) +- Separate configurations for colorized and json logs in Task SDK / Celery Executor (#51082) +- Enhanced task log viewer with virtualized rendering for improved performance on large logs (#50746) + +Doc Only Changes +"""""""""""""""" + +- Add dates for Limited Maintenance & EOL for Airflow 2.x (#50794) +- Add Apache Airflow setup instructions for Apple Silicon (#50179) +- Update recommendation for upgrade path to airflow 3 (#50318) +- Add "disappearing DAGs" section on FAQ doc (#49987) +- Update Airflow 3 migration guide with step about custom operators (#50871) (#50948) +- Use ``AssetAlias`` for alias in Asset ``Metadata`` example (#50768) +- Do not use outdated ``schedule_interval`` in tutorial dags (#50947) +- Add Airflow Version in Page Title (#50358) +- Fix callbacks docs (#50377) +- Updating operator extra links doc (#50197) +- Prune old Airflow versions from release notes (#50860) +- Fix types in config templates reference (#50792) +- Fix wrong import for ``PythonOperator`` in tutorial dag (#50962) +- Better structure of extras documentation (#50495) + Airflow 3.0.1 (2025-05-12) -------------------------- @@ -984,6 +1099,99 @@ thank everyone who helped shape this release through design discussions, code co community feedback. For full details, migration guidance, and upgrade best practices, refer to the official Upgrade Guide and join the conversation on the Airflow dev and user mailing lists. +Airflow 2.11.0 (2025-05-20) +--------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +``DeltaTriggerTimetable`` for trigger-based scheduling (#47074) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +This change introduces DeltaTriggerTimetable, a new built-in timetable that complements the existing suite of +Airflow timetables by supporting delta-based trigger schedules without relying on data intervals. + +Airflow currently has two major types of timetables: + - Data interval-based (e.g., ``CronDataIntervalTimetable``, ``DeltaDataIntervalTimetable``) + - Trigger-based (e.g., ``CronTriggerTimetable``) + +However, there was no equivalent trigger-based option for delta intervals like ``timedelta(days=1)``. +As a result, even simple schedules like ``schedule=timedelta(days=1)`` were interpreted through a data interval +lens—adding unnecessary complexity for users who don't care about upstream/downstream data dependencies. + +This feature is backported to Airflow 2.11.0 to help users begin transitioning before upgrading to Airflow 3.0. + + - In Airflow 2.11, ``schedule=timedelta(...)`` still defaults to ``DeltaDataIntervalTimetable``. + - A new config option ``[scheduler] create_delta_data_intervals`` (default: ``True``) allows opting in to ``DeltaTriggerTimetable``. + - In Airflow 3.0, this config defaults to ``False``, meaning ``DeltaTriggerTimetable`` becomes the default for timedelta schedules. + +By flipping this config in 2.11, users can preview and adopt the new scheduling behavior in advance — minimizing surprises during upgrade. + + +Consistent timing metrics across all backends (#39908, #43966) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Previously, Airflow reported timing metrics in milliseconds for ``StatsD`` but in seconds for other backends +such as ``OpenTelemetry`` and ``Datadog``. This inconsistency made it difficult to interpret or compare +timing metrics across systems. + +Airflow 2.11 introduces a new config option: + + - ``[metrics] timer_unit_consistency`` (default: ``False`` in 2.11, ``True`` and dropped in Airflow 3.0). + +When enabled, all timing metrics are consistently reported in milliseconds, regardless of the backend. + +This setting has become mandatory and always ``True`` in Airflow 3.0 (the config will be removed), so +enabling it in 2.11 allows users to migrate early and avoid surprises during upgrade. + +Ease migration to Airflow 3 +""""""""""""""""""""""""""" +This release introduces several changes to help users prepare for upgrading to Airflow 3: + + - All models using ``execution_date`` now also include a ``logical_date`` field. Airflow 3 drops ``execution_date`` entirely in favor of ``logical_date`` (#44283) + - Added ``airflow config lint`` and ``airflow config update`` commands in 2.11 to help audit and migrate configs for Airflow 3.0. (#45736, #50353, #46757) + +Python 3.8 support removed +"""""""""""""""""""""""""" +Support for Python 3.8 has been removed, as it has reached end-of-life. +Airflow 2.11 requires Python 3.9, 3.10, 3.11, or 3.12. + +New Features +"""""""""""" + +- Introduce ``DeltaTriggerTimetable`` (#47074) +- Backport ``airflow config update`` and ``airflow config lint`` changes to ease migration to Airflow 3 (#45736, #50353) +- Add link to show task in a DAG in DAG Dependencies view (#47721) +- Align timers and timing metrics (ms) across all metrics loggers (#39908, #43966) + +Bug Fixes +""""""""" + +- Don't resolve path for DAGs folder (#46877) +- Fix ``ti.log_url`` timestamp format from ``"%Y-%m-%dT%H:%M:%S%z"`` to ``"%Y-%m-%dT%H:%M:%S.%f%z"`` (#50306) +- Ensure that the generated ``airflow.cfg`` contains a random ``fernet_key`` and ``secret_key`` (#47755) +- Fixed setting ``rendered_map_index`` via internal api (#49057) +- Store rendered_map_index from ``TaskInstancePydantic`` into ``TaskInstance`` (#48571) +- Allow using ``log_url`` property on ``TaskInstancePydantic`` (Internal API) (#50560) +- Fix Trigger Form with Empty Object Default (#46872) +- Fix ``TypeError`` when deserializing task with ``execution_timeout`` set to ``None`` (#46822) +- Always populate mapped tasks (#46790) +- Ensure ``check_query_exists`` returns a bool (#46707) +- UI: ``/xcom/list`` got exception when applying filter on the ``value`` column (#46053) +- Allow to set note field via the experimental internal api (#47769) + +Miscellaneous +""""""""""""" + +- Add ``logical_date`` to models using ``execution_date`` (#44283) +- Drop support for Python 3.8 (#49980, #50015) +- Emit warning for deprecated ``BaseOperatorLink.get_link`` signature (#46448) + +Doc Only Changes +"""""""""""""""" +- Unquote executor ``airflow.cfg`` variable (#48084) +- Update ``XCom`` docs to show examples of pushing multiple ``XComs`` (#46284, #47068) + Airflow 2.10.5 (2025-02-10) --------------------------- @@ -4185,5334 +4393,6 @@ Doc only changes - Update docs: zip-like effect is now possible in task mapping (#26435) - changing to task decorator in docs from classic operator use (#25711) -Airflow 2.4.3 (2022-11-14) --------------------------- - -Significant Changes -^^^^^^^^^^^^^^^^^^^ - -Make ``RotatingFilehandler`` used in ``DagProcessor`` non-caching (#27223) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -In case you want to decrease cache memory when ``CONFIG_PROCESSOR_MANAGER_LOGGER=True``, and you have your local settings created before, -you can update ``processor_manager_handler`` to use ``airflow.utils.log.non_caching_file_handler.NonCachingRotatingFileHandler`` handler instead of ``logging.RotatingFileHandler``. - -Bug Fixes -^^^^^^^^^ -- Fix double logging with some task logging handler (#27591) -- Replace FAB url filtering function with Airflow's (#27576) -- Fix mini scheduler expansion of mapped task (#27506) -- ``SLAMiss`` is nullable and not always given back when pulling task instances (#27423) -- Fix behavior of ``_`` when searching for DAGs (#27448) -- Fix getting the ``dag/task`` ids from BaseExecutor (#27550) -- Fix SQLAlchemy primary key black-out error on DDRQ (#27538) -- Fix IntegrityError during webserver startup (#27297) -- Add case insensitive constraint to username (#27266) -- Fix python external template keys (#27256) -- Reduce extraneous task log requests (#27233) -- Make ``RotatingFilehandler`` used in ``DagProcessor`` non-caching (#27223) -- Listener: Set task on SQLAlchemy TaskInstance object (#27167) -- Fix dags list page auto-refresh & jump search null state (#27141) -- Set ``executor.job_id`` to ``BackfillJob.id`` for backfills (#27020) - -Misc/Internal -^^^^^^^^^^^^^ -- Bump loader-utils from ``1.4.0`` to ``1.4.1`` in ``/airflow/www`` (#27552) -- Reduce log level for k8s ``TCP_KEEPALIVE`` etc warnings (#26981) - -Doc only changes -^^^^^^^^^^^^^^^^ -- Use correct executable in docker compose docs (#27529) -- Fix wording in DAG Runs description (#27470) -- Document that ``KubernetesExecutor`` overwrites container args (#27450) -- Fix ``BaseOperator`` links (#27441) -- Correct timer units to seconds from milliseconds. (#27360) -- Add missed import in the Trigger Rules example (#27309) -- Update SLA wording to reflect it is relative to ``Dag Run`` start. (#27111) -- Add ``kerberos`` environment variables to the docs (#27028) - -Airflow 2.4.2 (2022-10-23) --------------------------- - -Significant Changes -^^^^^^^^^^^^^^^^^^^ - -Default for ``[webserver] expose_stacktrace`` changed to ``False`` (#27059) -""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The default for ``[webserver] expose_stacktrace`` has been set to ``False``, instead of ``True``. This means administrators must opt-in to expose tracebacks to end users. - -Bug Fixes -^^^^^^^^^ -- Make tracebacks opt-in (#27059) -- Add missing AUTOINC/SERIAL for FAB tables (#26885) -- Add separate error handler for 405(Method not allowed) errors (#26880) -- Don't re-patch pods that are already controlled by current worker (#26778) -- Handle mapped tasks in task duration chart (#26722) -- Fix task duration cumulative chart (#26717) -- Avoid 500 on dag redirect (#27064) -- Filter dataset dependency data on webserver (#27046) -- Remove double collection of dags in ``airflow dags reserialize`` (#27030) -- Fix auto refresh for graph view (#26926) -- Don't overwrite connection extra with invalid json (#27142) -- Fix next run dataset modal links (#26897) -- Change dag audit log sort by date from asc to desc (#26895) -- Bump min version of jinja2 (#26866) -- Add missing colors to ``state_color_mapping`` jinja global (#26822) -- Fix running debuggers inside ``airflow tasks test`` (#26806) -- Fix warning when using xcomarg dependencies (#26801) -- demote Removed state in priority for displaying task summaries (#26789) -- Ensure the log messages from operators during parsing go somewhere (#26779) -- Add restarting state to TaskState Enum in REST API (#26776) -- Allow retrieving error message from data.detail (#26762) -- Simplify origin string cleaning (#27143) -- Remove DAG parsing from StandardTaskRunner (#26750) -- Fix non-hidden cumulative chart on duration view (#26716) -- Remove TaskFail duplicates check (#26714) -- Fix airflow tasks run --local when dags_folder differs from that of processor (#26509) -- Fix yarn warning from d3-color (#27139) -- Fix version for a couple configurations (#26491) -- Revert "No grid auto-refresh for backfill dag runs (#25042)" (#26463) -- Retry on Airflow Schedule DAG Run DB Deadlock (#26347) - -Misc/Internal -^^^^^^^^^^^^^ -- Clean-ups around task-mapping code (#26879) -- Move user-facing string to template (#26815) -- add icon legend to datasets graph (#26781) -- Bump ``sphinx`` and ``sphinx-autoapi`` (#26743) -- Simplify ``RTIF.delete_old_records()`` (#26667) -- Bump FAB to ``4.1.4`` (#26393) - -Doc only changes -^^^^^^^^^^^^^^^^ -- Fixed triple quotes in task group example (#26829) -- Documentation fixes (#26819) -- make consistency on markup title string level (#26696) -- Add a note against use of top level code in timetable (#26649) -- Fix broken URL for ``docker-compose.yaml`` (#26726) - - -Airflow 2.4.1 (2022-09-30) --------------------------- - -Significant Changes -^^^^^^^^^^^^^^^^^^^ - -No significant changes. - -Bug Fixes -^^^^^^^^^ - -- When rendering template, unmap task in context (#26702) -- Fix scroll overflow for ConfirmDialog (#26681) -- Resolve deprecation warning re ``Table.exists()`` (#26616) -- Fix XComArg zip bug (#26636) -- Use COALESCE when ordering runs to handle NULL (#26626) -- Check user is active (#26635) -- No missing user warning for public admin (#26611) -- Allow MapXComArg to resolve after serialization (#26591) -- Resolve warning about DISTINCT ON query on dags view (#26608) -- Log warning when secret backend kwargs is invalid (#26580) -- Fix grid view log try numbers (#26556) -- Template rendering issue in passing ``templates_dict`` to task decorator (#26390) -- Fix Deferrable stuck as ``scheduled`` during backfill (#26205) -- Suppress SQLALCHEMY_TRACK_MODIFICATIONS warning in db init (#26617) -- Correctly set ``json_provider_class`` on Flask app so it uses our encoder (#26554) -- Fix WSGI root app (#26549) -- Fix deadlock when mapped task with removed upstream is rerun (#26518) -- ExecutorConfigType should be ``cacheable`` (#26498) -- Fix proper joining of the path for logs retrieved from celery workers (#26493) -- DAG Deps extends ``base_template`` (#26439) -- Don't update backfill run from the scheduler (#26342) - -Doc only changes -^^^^^^^^^^^^^^^^ - -- Clarify owner links document (#26515) -- Fix invalid RST in dataset concepts doc (#26434) -- Document the ``non-sensitive-only`` option for ``expose_config`` (#26507) -- Fix ``example_datasets`` dag names (#26495) -- Zip-like effect is now possible in task mapping (#26435) -- Use task decorator in docs instead of classic operators (#25711) - -Airflow 2.4.0 (2022-09-19) --------------------------- - -Significant Changes -^^^^^^^^^^^^^^^^^^^ - -Data-aware Scheduling and ``Dataset`` concept added to Airflow -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -New to this release of Airflow is the concept of Datasets to Airflow, and with it a new way of scheduling dags: -data-aware scheduling. - -This allows DAG runs to be automatically created as a result of a task "producing" a dataset. In some ways -this can be thought of as the inverse of ``TriggerDagRunOperator``, where instead of the producing DAG -controlling which DAGs get created, the consuming DAGs can "listen" for changes. - -A dataset is identified by a URI: - -.. code-block:: python - - from airflow import Dataset - - # The URI doesn't have to be absolute - dataset = Dataset(uri="my-dataset") - # Or you can use a scheme to show where it lives. - dataset2 = Dataset(uri="s3://bucket/prefix") - -To create a DAG that runs whenever a Dataset is updated use the new ``schedule`` parameter (see below) and -pass a list of 1 or more Datasets: - -.. code-block:: python - - with DAG(dag_id='dataset-consumer', schedule=[dataset]): - ... - -And to mark a task as producing a dataset pass the dataset(s) to the ``outlets`` attribute: - -.. code-block:: python - - @task(outlets=[dataset]) - def my_task(): ... - - - # Or for classic operators - BashOperator(task_id="update-ds", bash_command=..., outlets=[dataset]) - -If you have the producer and consumer in different files you do not need to use the same Dataset object, two -``Dataset()``\s created with the same URI are equal. - -Datasets represent the abstract concept of a dataset, and (for now) do not have any direct read or write -capability - in this release we are adding the foundational feature that we will build upon. - -For more info on Datasets please see `Datasets documentation `_. - -Expanded dynamic task mapping support -""""""""""""""""""""""""""""""""""""" - -Dynamic task mapping now includes support for ``expand_kwargs``, ``zip`` and ``map``. - -For more info on dynamic task mapping please see :doc:`/authoring-and-scheduling/dynamic-task-mapping`. - -DAGS used in a context manager no longer need to be assigned to a module variable (#23592) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Previously you had to assign a DAG to a module-level variable in order for Airflow to pick it up. For example this - - -.. code-block:: python - - with DAG(dag_id="example") as dag: - ... - - - @dag - def dag_maker(): ... - - - dag2 = dag_maker() - - -can become - -.. code-block:: python - - with DAG(dag_id="example"): - ... - - - @dag - def dag_maker(): ... - - - dag_maker() - -If you want to disable the behaviour for any reason then set ``auto_register=False`` on the dag: - -.. code-block:: python - - # This dag will not be picked up by Airflow as it's not assigned to a variable - with DAG(dag_id="example", auto_register=False): - ... - -Deprecation of ``schedule_interval`` and ``timetable`` arguments (#25410) -""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -We added new DAG argument ``schedule`` that can accept a cron expression, timedelta object, *timetable* object, or list of dataset objects. Arguments ``schedule_interval`` and ``timetable`` are deprecated. - -If you previously used the ``@daily`` cron preset, your DAG may have looked like this: - -.. code-block:: python - - with DAG( - dag_id="my_example", - start_date=datetime(2021, 1, 1), - schedule_interval="@daily", - ): - ... - -Going forward, you should use the ``schedule`` argument instead: - -.. code-block:: python - - with DAG( - dag_id="my_example", - start_date=datetime(2021, 1, 1), - schedule="@daily", - ): - ... - -The same is true if you used a custom timetable. Previously you would have used the ``timetable`` argument: - -.. code-block:: python - - with DAG( - dag_id="my_example", - start_date=datetime(2021, 1, 1), - timetable=EventsTimetable(event_dates=[pendulum.datetime(2022, 4, 5)]), - ): - ... - -Now you should use the ``schedule`` argument: - -.. code-block:: python - - with DAG( - dag_id="my_example", - start_date=datetime(2021, 1, 1), - schedule=EventsTimetable(event_dates=[pendulum.datetime(2022, 4, 5)]), - ): - ... - -Removal of experimental Smart Sensors (#25507) -"""""""""""""""""""""""""""""""""""""""""""""" - -Smart Sensors were added in 2.0 and deprecated in favor of Deferrable operators in 2.2, and have now been removed. - -``airflow.contrib`` packages and deprecated modules are dynamically generated (#26153, #26179, #26167) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The ``airflow.contrib`` packages and deprecated modules from Airflow 1.10 in ``airflow.hooks``, ``airflow.operators``, ``airflow.sensors`` packages are now dynamically generated modules and while users can continue using the deprecated contrib classes, they are no longer visible for static code check tools and will be reported as missing. It is recommended for the users to move to the non-deprecated classes. - -``DBApiHook`` and ``SQLSensor`` have moved (#24836) -""""""""""""""""""""""""""""""""""""""""""""""""""" - -``DBApiHook`` and ``SQLSensor`` have been moved to the ``apache-airflow-providers-common-sql`` provider. - -DAG runs sorting logic changed in grid view (#25090) -"""""""""""""""""""""""""""""""""""""""""""""""""""" - -The ordering of DAG runs in the grid view has been changed to be more "natural". -The new logic generally orders by data interval, but a custom ordering can be -applied by setting the DAG to use a custom timetable. - - -New Features -^^^^^^^^^^^^ -- Add Data-aware Scheduling (`AIP-48 `_) -- Add ``@task.short_circuit`` TaskFlow decorator (#25752) -- Make ``execution_date_or_run_id`` optional in ``tasks test`` command (#26114) -- Automatically register DAGs that are used in a context manager (#23592, #26398) -- Add option of sending DAG parser logs to stdout. (#25754) -- Support multiple ``DagProcessors`` parsing files from different locations. (#25935) -- Implement ``ExternalPythonOperator`` (#25780) -- Make execution_date optional for command ``dags test`` (#26111) -- Implement ``expand_kwargs()`` against a literal list (#25925) -- Add trigger rule tooltip (#26043) -- Add conf parameter to CLI for airflow dags test (#25900) -- Include scheduled slots in pools view (#26006) -- Add ``output`` property to ``MappedOperator`` (#25604) -- Add roles delete command to cli (#25854) -- Add Airflow specific warning classes (#25799) -- Add support for ``TaskGroup`` in ``ExternalTaskSensor`` (#24902) -- Add ``@task.kubernetes`` taskflow decorator (#25663) -- Add a way to import Airflow without side-effects (#25832) -- Let timetables control generated run_ids. (#25795) -- Allow per-timetable ordering override in grid view (#25633) -- Grid logs for mapped instances (#25610, #25621, #25611) -- Consolidate to one ``schedule`` param (#25410) -- DAG regex flag in backfill command (#23870) -- Adding support for owner links in the Dags view UI (#25280) -- Ability to clear a specific DAG Run's task instances via REST API (#23516) -- Possibility to document DAG with a separate markdown file (#25509) -- Add parsing context to DAG Parsing (#25161) -- Implement ``CronTriggerTimetable`` (#23662) -- Add option to mask sensitive data in UI configuration page (#25346) -- Create new databases from the ORM (#24156) -- Implement ``XComArg.zip(*xcom_args)`` (#25176) -- Introduce ``sla_miss`` metric (#23402) -- Implement ``map()`` semantic (#25085) -- Add override method to TaskGroupDecorator (#25160) -- Implement ``expand_kwargs()`` (#24989) -- Add parameter to turn off SQL query logging (#24570) -- Add ``DagWarning`` model, and a check for missing pools (#23317) -- Add Task Logs to Grid details panel (#24249) -- Added small health check server and endpoint in scheduler(#23905) -- Add built-in External Link for ``ExternalTaskMarker`` operator (#23964) -- Add default task retry delay config (#23861) -- Add clear DagRun endpoint. (#23451) -- Add support for timezone as string in cron interval timetable (#23279) -- Add auto-refresh to dags home page (#22900, #24770) - -Improvements -^^^^^^^^^^^^ - -- Add more weekday operator and sensor examples #26071 (#26098) -- Add subdir parameter to dags reserialize command (#26170) -- Update zombie message to be more descriptive (#26141) -- Only send an ``SlaCallbackRequest`` if the DAG is scheduled (#26089) -- Promote ``Operator.output`` more (#25617) -- Upgrade API files to typescript (#25098) -- Less ``hacky`` double-rendering prevention in mapped task (#25924) -- Improve Audit log (#25856) -- Remove mapped operator validation code (#25870) -- More ``DAG(schedule=...)`` improvements (#25648) -- Reduce ``operator_name`` dupe in serialized JSON (#25819) -- Make grid view group/mapped summary UI more consistent (#25723) -- Remove useless statement in ``task_group_to_grid`` (#25654) -- Add optional data interval to ``CronTriggerTimetable`` (#25503) -- Remove unused code in ``/grid`` endpoint (#25481) -- Add and document description fields (#25370) -- Improve Airflow logging for operator Jinja template processing (#25452) -- Update core example DAGs to use ``@task.branch`` decorator (#25242) -- Update DAG ``audit_log`` route (#25415) -- Change stdout and stderr access mode to append in commands (#25253) -- Remove ``getTasks`` from Grid view (#25359) -- Improve taskflow type hints with ParamSpec (#25173) -- Use tables in grid details panes (#25258) -- Explicitly list ``@dag`` arguments (#25044) -- More typing in ``SchedulerJob`` and ``TaskInstance`` (#24912) -- Patch ``getfqdn`` with more resilient version (#24981) -- Replace all ``NBSP`` characters by ``whitespaces`` (#24797) -- Re-serialize all DAGs on ``airflow db upgrade`` (#24518) -- Rework contract of try_adopt_task_instances method (#23188) -- Make ``expand()`` error vague so it's not misleading (#24018) -- Add enum validation for ``[webserver]analytics_tool`` (#24032) -- Add ``dttm`` searchable field in audit log (#23794) -- Allow more parameters to be piped through via ``execute_in_subprocess`` (#23286) -- Use ``func.count`` to count rows (#23657) -- Remove stale serialized dags (#22917) -- AIP45 Remove dag parsing in airflow run local (#21877) -- Add support for queued state in DagRun update endpoint. (#23481) -- Add fields to dagrun endpoint (#23440) -- Use ``sql_alchemy_conn`` for celery result backend when ``result_backend`` is not set (#24496) - -Bug Fixes -^^^^^^^^^ - -- Have consistent types between the ORM and the migration files (#24044, #25869) -- Disallow any dag tags longer than 100 char (#25196) -- Add the dag_id to ``AirflowDagCycleException`` message (#26204) -- Properly build URL to retrieve logs independently from system (#26337) -- For worker log servers only bind to IPV6 when dual stack is available (#26222) -- Fix ``TaskInstance.task`` not defined before ``handle_failure`` (#26040) -- Undo secrets backend config caching (#26223) -- Fix faulty executor config serialization logic (#26191) -- Show ``DAGs`` and ``Datasets`` menu links based on role permission (#26183) -- Allow setting ``TaskGroup`` tooltip via function docstring (#26028) -- Fix RecursionError on graph view of a DAG with many tasks (#26175) -- Fix backfill occasional deadlocking (#26161) -- Fix ``DagRun.start_date`` not set during backfill with ``--reset-dagruns`` True (#26135) -- Use label instead of id for dynamic task labels in graph (#26108) -- Don't fail DagRun when leaf ``mapped_task`` is SKIPPED (#25995) -- Add group prefix to decorated mapped task (#26081) -- Fix UI flash when triggering with dup logical date (#26094) -- Fix Make items nullable for ``TaskInstance`` related endpoints to avoid API errors (#26076) -- Fix ``BranchDateTimeOperator`` to be ``timezone-awreness-insensitive`` (#25944) -- Fix legacy timetable schedule interval params (#25999) -- Fix response schema for ``list-mapped-task-instance`` (#25965) -- Properly check the existence of missing mapped TIs (#25788) -- Fix broken auto-refresh on grid view (#25950) -- Use per-timetable ordering in grid UI (#25880) -- Rewrite recursion when parsing DAG into iteration (#25898) -- Find cross-group tasks in ``iter_mapped_dependants`` (#25793) -- Fail task if mapping upstream fails (#25757) -- Support ``/`` in variable get endpoint (#25774) -- Use cfg default_wrap value for grid logs (#25731) -- Add origin request args when triggering a run (#25729) -- Operator name separate from class (#22834) -- Fix incorrect data interval alignment due to assumption on input time alignment (#22658) -- Return None if an ``XComArg`` fails to resolve (#25661) -- Correct ``json`` arg help in ``airflow variables set`` command (#25726) -- Added MySQL index hint to use ``ti_state`` on ``find_zombies`` query (#25725) -- Only excluded actually expanded fields from render (#25599) -- Grid, fix toast for ``axios`` errors (#25703) -- Fix UI redirect (#26409) -- Require dag_id arg for dags list-runs (#26357) -- Check for queued states for dags auto-refresh (#25695) -- Fix upgrade code for the ``dag_owner_attributes`` table (#25579) -- Add map index to task logs api (#25568) -- Ensure that zombie tasks for dags with errors get cleaned up (#25550) -- Make extra link work in UI (#25500) -- Sync up plugin API schema and definition (#25524) -- First/last names can be empty (#25476) -- Refactor DAG pages to be consistent (#25402) -- Check ``expand_kwargs()`` input type before unmapping (#25355) -- Filter XCOM by key when calculating map lengths (#24530) -- Fix ``ExternalTaskSensor`` not working with dynamic task (#25215) -- Added exception catching to send default email if template file raises any exception (#24943) -- Bring ``MappedOperator`` members in sync with ``BaseOperator`` (#24034) - - -Misc/Internal -^^^^^^^^^^^^^ - -- Add automatically generated ``ERD`` schema for the ``MetaData`` DB (#26217) -- Mark serialization functions as internal (#26193) -- Remove remaining deprecated classes and replace them with ``PEP562`` (#26167) -- Move ``dag_edges`` and ``task_group_to_dict`` to corresponding util modules (#26212) -- Lazily import many modules to improve import speed (#24486, #26239) -- FIX Incorrect typing information (#26077) -- Add missing contrib classes to deprecated dictionaries (#26179) -- Re-configure/connect the ``ORM`` after forking to run a DAG processor (#26216) -- Remove cattrs from lineage processing. (#26134) -- Removed deprecated contrib files and replace them with ``PEP-562`` getattr (#26153) -- Make ``BaseSerialization.serialize`` "public" to other classes. (#26142) -- Change the template to use human readable task_instance description (#25960) -- Bump ``moment-timezone`` from ``0.5.34`` to ``0.5.35`` in ``/airflow/www`` (#26080) -- Fix Flask deprecation warning (#25753) -- Add ``CamelCase`` to generated operations types (#25887) -- Fix migration issues and tighten the CI upgrade/downgrade test (#25869) -- Fix type annotations in ``SkipMixin`` (#25864) -- Workaround setuptools editable packages path issue (#25848) -- Bump ``undici`` from ``5.8.0 to 5.9.1`` in /airflow/www (#25801) -- Add custom_operator_name attr to ``_BranchPythonDecoratedOperator`` (#25783) -- Clarify ``filename_template`` deprecation message (#25749) -- Use ``ParamSpec`` to replace ``...`` in Callable (#25658) -- Remove deprecated modules (#25543) -- Documentation on task mapping additions (#24489) -- Remove Smart Sensors (#25507) -- Fix ``elasticsearch`` test config to avoid warning on deprecated template (#25520) -- Bump ``terser`` from ``4.8.0 to 4.8.1`` in /airflow/ui (#25178) -- Generate ``typescript`` types from rest ``API`` docs (#25123) -- Upgrade utils files to ``typescript`` (#25089) -- Upgrade remaining context file to ``typescript``. (#25096) -- Migrate files to ``ts`` (#25267) -- Upgrade grid Table component to ``ts.`` (#25074) -- Skip mapping against mapped ``ti`` if it returns None (#25047) -- Refactor ``js`` file structure (#25003) -- Move mapped kwargs introspection to separate type (#24971) -- Only assert stuff for mypy when type checking (#24937) -- Bump ``moment`` from ``2.29.3 to 2.29.4`` in ``/airflow/www`` (#24885) -- Remove "bad characters" from our codebase (#24841) -- Remove ``xcom_push`` flag from ``BashOperator`` (#24824) -- Move Flask hook registration to end of file (#24776) -- Upgrade more javascript files to ``typescript`` (#24715) -- Clean up task decorator type hints and docstrings (#24667) -- Preserve original order of providers' connection extra fields in UI (#24425) -- Rename ``charts.css`` to ``chart.css`` (#24531) -- Rename ``grid.css`` to ``chart.css`` (#24529) -- Misc: create new process group by ``set_new_process_group`` utility (#24371) -- Airflow UI fix Prototype Pollution (#24201) -- Bump ``moto`` version (#24222) -- Remove unused ``[github_enterprise]`` from ref docs (#24033) -- Clean up ``f-strings`` in logging calls (#23597) -- Add limit for ``JPype1`` (#23847) -- Simply json responses (#25518) -- Add min attrs version (#26408) - -Doc only changes -^^^^^^^^^^^^^^^^ -- Add url prefix setting for ``Celery`` Flower (#25986) -- Updating deprecated configuration in examples (#26037) -- Fix wrong link for taskflow tutorial (#26007) -- Reorganize tutorials into a section (#25890) -- Fix concept doc for dynamic task map (#26002) -- Update code examples from "classic" operators to taskflow (#25845, #25657) -- Add instructions on manually fixing ``MySQL`` Charset problems (#25938) -- Prefer the local Quick Start in docs (#25888) -- Fix broken link to ``Trigger Rules`` (#25840) -- Improve docker documentation (#25735) -- Correctly link to Dag parsing context in docs (#25722) -- Add note on ``task_instance_mutation_hook`` usage (#25607) -- Note that TaskFlow API automatically passes data between tasks (#25577) -- Update DAG run to clarify when a DAG actually runs (#25290) -- Update tutorial docs to include a definition of operators (#25012) -- Rewrite the Airflow documentation home page (#24795) -- Fix ``task-generated mapping`` example (#23424) -- Add note on subtle logical date change in ``2.2.0`` (#24413) -- Add missing import in best-practices code example (#25391) - - - -Airflow 2.3.4 (2022-08-23) --------------------------- - -Significant Changes -^^^^^^^^^^^^^^^^^^^ - -Added new config ``[logging]log_formatter_class`` to fix timezone display for logs on UI (#24811) -""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -If you are using a custom Formatter subclass in your ``[logging]logging_config_class``, please inherit from ``airflow.utils.log.timezone_aware.TimezoneAware`` instead of ``logging.Formatter``. -For example, in your ``custom_config.py``: - -.. code-block:: python - - from airflow.utils.log.timezone_aware import TimezoneAware - - - # before - class YourCustomFormatter(logging.Formatter): ... - - - # after - class YourCustomFormatter(TimezoneAware): ... - - - AIRFLOW_FORMATTER = LOGGING_CONFIG["formatters"]["airflow"] - AIRFLOW_FORMATTER["class"] = "somewhere.your.custom_config.YourCustomFormatter" - # or use TimezoneAware class directly. If you don't have custom Formatter. - AIRFLOW_FORMATTER["class"] = "airflow.utils.log.timezone_aware.TimezoneAware" - -Bug Fixes -^^^^^^^^^ - -- Disable ``attrs`` state management on ``MappedOperator`` (#24772) -- Serialize ``pod_override`` to JSON before pickling ``executor_config`` (#24356) -- Fix ``pid`` check (#24636) -- Rotate session id during login (#25771) -- Fix mapped sensor with reschedule mode (#25594) -- Cache the custom secrets backend so the same instance gets reused (#25556) -- Add right padding (#25554) -- Fix reducing mapped length of a mapped task at runtime after a clear (#25531) -- Fix ``airflow db reset`` when dangling tables exist (#25441) -- Change ``disable_verify_ssl`` behaviour (#25023) -- Set default task group in dag.add_task method (#25000) -- Removed interfering force of index. (#25404) -- Remove useless logging line (#25347) -- Adding mysql index hint to use index on ``task_instance.state`` in critical section query (#25673) -- Configurable umask to all daemonized processes. (#25664) -- Fix the errors raised when None is passed to template filters (#25593) -- Allow wildcarded CORS origins (#25553) -- Fix "This Session's transaction has been rolled back" (#25532) -- Fix Serialization error in ``TaskCallbackRequest`` (#25471) -- fix - resolve bash by absolute path (#25331) -- Add ``__repr__`` to ParamsDict class (#25305) -- Only load distribution of a name once (#25296) -- convert ``TimeSensorAsync`` ``target_time`` to utc on call time (#25221) -- call ``updateNodeLabels`` after ``expandGroup`` (#25217) -- Stop SLA callbacks gazumping other callbacks and DOS'ing the ``DagProcessorManager`` queue (#25147) -- Fix ``invalidateQueries`` call (#25097) -- ``airflow/www/package.json``: Add name, version fields. (#25065) -- No grid auto-refresh for backfill dag runs (#25042) -- Fix tag link on dag detail page (#24918) -- Fix zombie task handling with multiple schedulers (#24906) -- Bind log server on worker to ``IPv6`` address (#24755) (#24846) -- Add ``%z`` for ``%(asctime)s`` to fix timezone for logs on UI (#24811) -- ``TriggerDagRunOperator.operator_extra_links`` is attr (#24676) -- Send DAG timeout callbacks to processor outside of ``prohibit_commit`` (#24366) -- Don't rely on current ORM structure for db clean command (#23574) -- Clear next method when clearing TIs (#23929) -- Two typing fixes (#25690) - -Doc only changes -^^^^^^^^^^^^^^^^ - -- Update set-up-database.rst (#24983) -- Fix syntax in mysql setup documentation (#24893 (#24939) -- Note how DAG policy works with default_args (#24804) -- Update PythonVirtualenvOperator Howto (#24782) -- Doc: Add hyperlinks to Github PRs for Release Notes (#24532) - -Misc/Internal -^^^^^^^^^^^^^ - -- Remove depreciation warning when use default remote tasks logging handlers (#25764) -- clearer method name in scheduler_job.py (#23702) -- Bump cattrs version (#25689) -- Include missing mention of ``external_executor_id`` in ``sql_engine_collation_for_ids`` docs (#25197) -- Refactor ``DR.task_instance_scheduling_decisions`` (#24774) -- Sort operator extra links (#24992) -- Extends ``resolve_xcom_backend`` function level documentation (#24965) -- Upgrade FAB to 4.1.3 (#24884) -- Limit Flask to <2.3 in the wake of 2.2 breaking our tests (#25511) -- Limit astroid version to < 2.12 (#24982) -- Move javascript compilation to host (#25169) -- Bump typing-extensions and mypy for ParamSpec (#25088) - - -Airflow 2.3.3 (2022-07-09) --------------------------- - -Significant Changes -^^^^^^^^^^^^^^^^^^^ - -We've upgraded Flask App Builder to a major version 4.* (#24399) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Flask App Builder is one of the important components of Airflow Webserver, as -it uses a lot of dependencies that are essential to run the webserver and integrate it -in enterprise environments - especially authentication. - -The FAB 4.* upgrades a number of dependencies to major releases, which upgrades them to versions -that have a number of security issues fixed. A lot of tests were performed to bring the dependencies -in a backwards-compatible way, however the dependencies themselves implement breaking changes in their -internals so it might be that some of those changes might impact the users in case they are using the -libraries for their own purposes. - -One important change that you likely will need to apply to Oauth configuration is to add -``server_metadata_url`` or ``jwks_uri`` and you can read about it more -in `this issue `_. - -Here is the list of breaking changes in dependencies that comes together with FAB 4: - - * ``Flask`` from 1.X to 2.X `breaking changes `__ - - * ``flask-jwt-extended`` 3.X to 4.X `breaking changes: `__ - - * ``Jinja2`` 2.X to 3.X `breaking changes: `__ - - * ``Werkzeug`` 1.X to 2.X `breaking changes `__ - - * ``pyJWT`` 1.X to 2.X `breaking changes: `__ - - * ``Click`` 7.X to 8.X `breaking changes: `__ - - * ``itsdangerous`` 1.X to 2.X `breaking changes `__ - -Bug Fixes -^^^^^^^^^ - -- Fix exception in mini task scheduler (#24865) -- Fix cycle bug with attaching label to task group (#24847) -- Fix timestamp defaults for ``sensorinstance`` (#24638) -- Move fallible ``ti.task.dag`` assignment back inside ``try/except`` block (#24533) (#24592) -- Add missing types to ``FSHook`` (#24470) -- Mask secrets in ``stdout`` for ``airflow tasks test`` (#24362) -- ``DebugExecutor`` use ``ti.run()`` instead of ``ti._run_raw_task`` (#24357) -- Fix bugs in ``URI`` constructor for ``MySQL`` connection (#24320) -- Missing ``scheduleinterval`` nullable true added in ``openapi`` (#24253) -- Unify ``return_code`` interface for task runner (#24093) -- Handle occasional deadlocks in trigger with retries (#24071) -- Remove special serde logic for mapped ``op_kwargs`` (#23860) -- ``ExternalTaskSensor`` respects ``soft_fail`` if the external task enters a ``failed_state`` (#23647) -- Fix ``StatD`` timing metric units (#21106) -- Add ``cache_ok`` flag to sqlalchemy TypeDecorators. (#24499) -- Allow for ``LOGGING_LEVEL=DEBUG`` (#23360) -- Fix grid date ticks (#24738) -- Debounce status highlighting in Grid view (#24710) -- Fix Grid vertical scrolling (#24684) -- don't try to render child rows for closed groups (#24637) -- Do not calculate grid root instances (#24528) -- Maintain grid view selection on filtering upstream (#23779) -- Speed up ``grid_data`` endpoint by 10x (#24284) -- Apply per-run log templates to log handlers (#24153) -- Don't crash scheduler if exec config has old k8s objects (#24117) -- ``TI.log_url`` fix for ``map_index`` (#24335) -- Fix migration ``0080_2_0_2`` - Replace null values before setting column not null (#24585) -- Patch ``sql_alchemy_conn`` if old Postgres schemes used (#24569) -- Seed ``log_template`` table (#24511) -- Fix deprecated ``log_id_template`` value (#24506) -- Fix toast messages (#24505) -- Add indexes for CASCADE deletes for ``task_instance`` (#24488) -- Return empty dict if Pod JSON encoding fails (#24478) -- Improve grid rendering performance with a custom tooltip (#24417, #24449) -- Check for ``run_id`` for grid group summaries (#24327) -- Optimize calendar view for cron scheduled DAGs (#24262) -- Use ``get_hostname`` instead of ``socket.getfqdn`` (#24260) -- Check that edge nodes actually exist (#24166) -- Fix ``useTasks`` crash on error (#24152) -- Do not fail re-queued TIs (#23846) -- Reduce grid view API calls (#24083) -- Rename Permissions to Permission Pairs. (#24065) -- Replace ``use_task_execution_date`` with ``use_task_logical_date`` (#23983) -- Grid fix details button truncated and small UI tweaks (#23934) -- Add TaskInstance State ``REMOVED`` to finished states and success states (#23797) -- Fix mapped task immutability after clear (#23667) -- Fix permission issue for dag that has dot in name (#23510) -- Fix closing connection ``dbapi.get_pandas_df`` (#23452) -- Check bag DAG ``schedule_interval`` match timetable (#23113) -- Parse error for task added to multiple groups (#23071) -- Fix flaky order of returned dag runs (#24405) -- Migrate ``jsx`` files that affect run/task selection to ``tsx`` (#24509) -- Fix links to sources for examples (#24386) -- Set proper ``Content-Type`` and ``chartset`` on ``grid_data`` endpoint (#24375) - -Doc only changes -^^^^^^^^^^^^^^^^ - -- Update templates doc to mention ``extras`` and format Airflow ``Vars`` / ``Conns`` (#24735) -- Document built in Timetables (#23099) -- Alphabetizes two tables (#23923) -- Clarify that users should not use Maria DB (#24556) -- Add imports to deferring code samples (#24544) -- Add note about image regeneration in June 2022 (#24524) -- Small cleanup of ``get_current_context()`` chapter (#24482) -- Fix default 2.2.5 ``log_id_template`` (#24455) -- Update description of installing providers separately from core (#24454) -- Mention context variables and logging (#24304) - -Misc/Internal -^^^^^^^^^^^^^ - -- Remove internet explorer support (#24495) -- Removing magic status code numbers from ``api_connexion`` (#24050) -- Upgrade FAB to ``4.1.2`` (#24619) -- Switch Markdown engine to ``markdown-it-py`` (#19702) -- Update ``rich`` to latest version across the board. (#24186) -- Get rid of ``TimedJSONWebSignatureSerializer`` (#24519) -- Update flask-appbuilder ``authlib``/ ``oauth`` dependency (#24516) -- Upgrade to ``webpack`` 5 (#24485) -- Add ``typescript`` (#24337) -- The JWT claims in the request to retrieve logs have been standardized: we use ``nbf`` and ``aud`` claims for - maturity and audience of the requests. Also "filename" payload field is used to keep log name. (#24519) -- Address all ``yarn`` test warnings (#24722) -- Upgrade to react 18 and chakra 2 (#24430) -- Refactor ``DagRun.verify_integrity`` (#24114) -- Upgrade FAB to ``4.1.1`` (#24399) -- We now need at least ``Flask-WTF 0.15`` (#24621) - - -Airflow 2.3.2 (2022-06-04) --------------------------- - -No significant changes. - -Bug Fixes -^^^^^^^^^ - -- Run the ``check_migration`` loop at least once -- Fix grid view for mapped tasks (#24059) -- Icons in grid view for different DAG run types (#23970) -- Faster grid view (#23951) -- Disallow calling expand with no arguments (#23463) -- Add missing ``is_mapped`` field to Task response. (#23319) -- DagFileProcessorManager: Start a new process group only if current process not a session leader (#23872) -- Mask sensitive values for not-yet-running TIs (#23807) -- Add cascade to ``dag_tag`` to ``dag`` foreign key (#23444) -- Use ``--subdir`` argument value for standalone dag processor. (#23864) -- Highlight task states by hovering on legend row (#23678) -- Fix and speed up grid view (#23947) -- Prevent UI from crashing if grid task instances are null (#23939) -- Remove redundant register exit signals in ``dag-processor`` command (#23886) -- Add ``__wrapped__`` property to ``_TaskDecorator`` (#23830) -- Fix UnboundLocalError when ``sql`` is empty list in DbApiHook (#23816) -- Enable clicking on DAG owner in autocomplete dropdown (#23804) -- Simplify flash message for ``_airflow_moved`` tables (#23635) -- Exclude missing tasks from the gantt view (#23627) - -Doc only changes -^^^^^^^^^^^^^^^^ - -- Add column names for DB Migration Reference (#23853) - -Misc/Internal -^^^^^^^^^^^^^ - -- Remove pinning for xmltodict (#23992) - - -Airflow 2.3.1 (2022-05-25) --------------------------- - -Significant Changes -^^^^^^^^^^^^^^^^^^^ -No significant changes. - -Bug Fixes -^^^^^^^^^ - -- Automatically reschedule stalled queued tasks in ``CeleryExecutor`` (#23690) -- Fix expand/collapse all buttons (#23590) -- Grid view status filters (#23392) -- Expand/collapse all groups (#23487) -- Fix retrieval of deprecated non-config values (#23723) -- Fix secrets rendered in UI when task is not executed. (#22754) -- Fix provider import error matching (#23825) -- Fix regression in ignoring symlinks (#23535) -- Fix ``dag-processor`` fetch metadata database config (#23575) -- Fix auto upstream dep when expanding non-templated field (#23771) -- Fix task log is not captured (#23684) -- Add ``reschedule`` to the serialized fields for the ``BaseSensorOperator`` (#23674) -- Modify db clean to also catch the ProgrammingError exception (#23699) -- Remove titles from link buttons (#23736) -- Fix grid details header text overlap (#23728) -- Ensure ``execution_timeout`` as timedelta (#23655) -- Don't run pre-migration checks for downgrade (#23634) -- Add index for event column in log table (#23625) -- Implement ``send_callback`` method for ``CeleryKubernetesExecutor`` and ``LocalKubernetesExecutor`` (#23617) -- Fix ``PythonVirtualenvOperator`` templated_fields (#23559) -- Apply specific ID collation to ``root_dag_id`` too (#23536) -- Prevent ``KubernetesJobWatcher`` getting stuck on resource too old (#23521) -- Fix scheduler crash when expanding with mapped task that returned none (#23486) -- Fix broken dagrun links when many runs start at the same time (#23462) -- Fix: Exception when parsing log #20966 (#23301) -- Handle invalid date parsing in webserver views. (#23161) -- Pools with negative open slots should not block other pools (#23143) -- Move around overflow, position and padding (#23044) -- Change approach to finding bad rows to LEFT OUTER JOIN. (#23528) -- Only count bad refs when ``moved`` table exists (#23491) -- Visually distinguish task group summary (#23488) -- Remove color change for highly nested groups (#23482) -- Optimize 2.3.0 pre-upgrade check queries (#23458) -- Add backward compatibility for ``core__sql_alchemy_conn__cmd`` (#23441) -- Fix literal cross product expansion (#23434) -- Fix broken task instance link in xcom list (#23367) -- Fix connection test button (#23345) -- fix cli ``airflow dags show`` for mapped operator (#23339) -- Hide some task instance attributes (#23338) -- Don't show grid actions if server would reject with permission denied (#23332) -- Use run_id for ``ti.mark_success_url`` (#23330) -- Fix update user auth stats (#23314) -- Use ``