diff --git a/.github/.pre-commit-config.yaml b/.github/.pre-commit-config.yaml index 909f0c1cdca3c..69bb3b8c3c8df 100644 --- a/.github/.pre-commit-config.yaml +++ b/.github/.pre-commit-config.yaml @@ -18,7 +18,7 @@ default_stages: [manual] default_language_version: python: python311 -minimum_pre_commit_version: '3.2.0' +minimum_prek_version: '0.0.26' repos: - repo: https://github.com/eclipse-csi/octopin rev: 21360742e352e87450f99e180fdfc2cf774a72a3 diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-prek/action.yml similarity index 52% rename from .github/actions/install-pre-commit/action.yml rename to .github/actions/install-prek/action.yml index b61154b5098b0..594dab1149d13 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-prek/action.yml @@ -16,38 +16,33 @@ # under the License. # --- -name: 'Install pre-commit' -description: 'Installs pre-commit and related packages' +name: 'Install prek' +description: 'Installs prek and related packages' inputs: python-version: description: 'Python version to use' default: "3.9" uv-version: description: 'uv version to use' - default: "0.8.8" # Keep this comment to allow automatic replacement of uv version - pre-commit-version: - description: 'pre-commit version to use' - default: "4.3.0" # Keep this comment to allow automatic replacement of pre-commit version - pre-commit-uv-version: - description: 'pre-commit-uv version to use' - default: "4.1.4" # Keep this comment to allow automatic replacement of pre-commit-uv version - skip-pre-commits: - description: "Skip some pre-commits from installation" + default: "0.8.11" # Keep this comment to allow automatic replacement of uv version + prek-version: + description: 'prek version to use' + default: "0.0.26" # Keep this comment to allow automatic replacement of prek version + skip-prek-hooks: + description: "Skip some prek hooks from installation" default: "" runs: using: "composite" steps: - - name: Install pre-commit, uv, and pre-commit-uv + - name: Install prek and uv shell: bash env: UV_VERSION: ${{inputs.uv-version}} - PRE_COMMIT_VERSION: ${{inputs.pre-commit-version}} - PRE_COMMIT_UV_VERSION: ${{inputs.pre-commit-uv-version}} - SKIP: ${{ inputs.skip-pre-commits }} + PREK_VERSION: ${{inputs.prek-version}} + SKIP: ${{ inputs.skip-prek-hooks }} run: | pip install uv==${UV_VERSION} || true - uv tool install pre-commit==${PRE_COMMIT_VERSION} --with uv==${UV_VERSION} \ - --with pre-commit-uv==${PRE_COMMIT_UV_VERSION} + uv tool install prek==${PREK_VERSION} --with uv==${UV_VERSION} working-directory: ${{ github.workspace }} # We need to use tar file with archive to restore all the permissions and symlinks - name: "Delete ~.cache" @@ -59,36 +54,36 @@ runs: rm -rf ~/.cache echo shell: bash - - name: "Restore pre-commit cache" + - name: "Restore prek cache" uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468 with: - key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + key: cache-prek-v5-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} path: /tmp/ - id: restore-pre-commit-cache - - name: "Check if pre-commit cache tarball exists" + id: restore-prek-cache + - name: "Check if prek cache tarball exists" shell: bash run: | - if [ -f /tmp/cache-pre-commit.tar.gz ]; then - echo "✅ Cache tarball found: /tmp/cache-pre-commit.tar.gz" + if [ -f /tmp/cache-prek.tar.gz ]; then + echo "✅ Cache tarball found: /tmp/cache-prek.tar.gz" else - echo "❌ Cache tarball missing. Expected /tmp/cache-pre-commit.tar.gz" + echo "❌ Cache tarball missing. Expected /tmp/cache-prek.tar.gz" exit 1 fi - if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + if: steps.restore-prek-cache.outputs.stash-hit == 'true' - name: "Restore .cache from the tar file" - run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz + run: tar -C ~ -xzf /tmp/cache-prek.tar.gz shell: bash - if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + if: steps.restore-prek-cache.outputs.stash-hit == 'true' - name: "Show restored files" run: | echo "Restored files" du ~/ --max-depth=2 echo shell: bash - if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' - - name: Install pre-commit hooks + if: steps.restore-prek-cache.outputs.stash-hit == 'true' + - name: Install prek hooks shell: bash - run: pre-commit install-hooks || (cat ~/.cache/pre-commit/pre-commit.log && exit 1) + run: prek install-hooks || (cat ~/.cache/prek/prek.log && exit 1) working-directory: ${{ github.workspace }} env: - SKIP: ${{ inputs.skip-pre-commits }} + SKIP: ${{ inputs.skip-prek-hooks }} diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 7cf001621d3c6..4b62c2fba8611 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -479,8 +479,8 @@ firstPRWelcomeComment: > Here are some useful points: - - Pay attention to the quality of your code (ruff, mypy and type annotations). Our [pre-commits]( - https://github.com/apache/airflow/blob/main/contributing-docs/08_static_code_checks.rst#prerequisites-for-pre-commit-hooks) + - Pay attention to the quality of your code (ruff, mypy and type annotations). Our [prek-hooks]( + https://github.com/apache/airflow/blob/main/contributing-docs/08_static_code_checks.rst#prerequisites-for-prek-hooks) will help you with that. - In case of a new feature add useful documentation (in docstrings or in `docs/` directory). diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index f3982389301e6..d289f38d1a358 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -48,8 +48,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to upgrade to newer dependencies (true/false)" required: true type: string - skip-pre-commits: - description: "Whether to skip pre-commits (true/false)" + skip-prek-hooks: + description: "Whether to skip prek hooks (true/false)" required: true type: string docker-cache: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index d6ea2b417e8e6..acb62059e3b0a 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -40,8 +40,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to run only basic checks (true/false)" required: true type: string - skip-pre-commits: - description: "Whether to skip pre-commits (true/false)" + skip-prek-hooks: + description: "Whether to skip prek hooks (true/false)" required: true type: string default-python-version: @@ -150,13 +150,13 @@ jobs: retention-days: '2' if: steps.restore-eslint-cache-simple-am-ui.outputs.stash-hit != 'true' - install-pre-commit: + install-prek: timeout-minutes: 5 - name: "Install pre-commit for cache" + name: "Install prek for cache" runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - SKIP: ${{ inputs.skip-pre-commits }} + SKIP: ${{ inputs.skip-prek-hooks }} steps: - name: "Cleanup repo" shell: bash @@ -168,21 +168,21 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{steps.breeze.outputs.host-python-version}} - skip-pre-commits: ${{ inputs.skip-pre-commits }} + skip-prek-hooks: ${{ inputs.skip-prek-hooks }} # Those checks are run if no image needs to be built for checks. This is for simple changes that # Do not touch any of the python code or any of the important files that might require building - # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host + # The CI Docker image and they can be run entirely using the prek virtual environments on host static-checks-basic-checks-only: timeout-minutes: 30 name: "Static checks: basic checks only" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-pre-commit + needs: install-prek if: inputs.basic-checks-only == 'true' steps: - name: "Cleanup repo" @@ -197,12 +197,12 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{ steps.breeze.outputs.host-python-version }} - skip-pre-commits: ${{ inputs.skip-pre-commits }} + skip-prek-hooks: ${{ inputs.skip-prek-hooks }} - name: Fetch incoming commit ${{ github.sha }} with its parent uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -211,12 +211,12 @@ jobs: persist-credentials: false - name: "Static checks: basic checks only" run: > - breeze static-checks --show-diff-on-failure --color always --initialize-environment - --skip-image-upgrade-check --commit-ref "${{ github.sha }}" + prek --show-diff-on-failure --color always + --from-ref "${{ github.sha }}" --to-ref "${{ github.sha }}" env: VERBOSE: "false" - SKIP_BREEZE_PRE_COMMITS: "true" - SKIP: ${{ inputs.skip-pre-commits }} + SKIP_BREEZE_PREK_HOOKS: "true" + SKIP: ${{ inputs.skip-prek-hooks }} COLUMNS: "202" test-git-clone-on-windows: @@ -234,7 +234,7 @@ jobs: timeout-minutes: 45 name: "Upgrade checks" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-pre-commit + needs: install-prek env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" if: inputs.canary-run == 'true' @@ -251,28 +251,28 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{steps.breeze.outputs.host-python-version}} - skip-pre-commits: ${{ inputs.skip-pre-commits }} - - name: "Autoupdate all pre-commits" - run: pre-commit autoupdate --freeze - - name: "Autoupdate Lucas-C/pre-commit-hooks to bleeding edge" - run: pre-commit autoupdate --bleeding-edge --freeze --repo https://github.com/Lucas-C/pre-commit-hooks + skip-prek-hooks: ${{ inputs.skip-prek-hooks }} + - name: "Autoupdate all prek hooks" + run: prek autoupdate --freeze + - name: "Autoupdate Lucas-C hooks to bleeding edge" + run: prek autoupdate --bleeding-edge --freeze --repo https://github.com/Lucas-C/pre-commit-hooks - name: "Check if there are any changes in pre-commit hooks" run: git diff --exit-code - name: "Run automated upgrade for black" run: > - pre-commit run + prek --all-files --show-diff-on-failure --color always --verbose --hook-stage manual update-black-version if: always() - name: "Run automated upgrade for chart dependencies" run: > - pre-commit run + prek --all-files --show-diff-on-failure --color always --verbose --hook-stage manual update-chart-dependencies @@ -282,30 +282,30 @@ jobs: # get notified about it - until it stabilizes in 1.* version - name: "Run automated upgrade for uv (not failing - just informational)" run: > - pre-commit run + prek --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual update-installers-and-pre-commit || true + --hook-stage manual upgrade-important-versions || true if: always() env: UPGRADE_UV: "true" UPGRADE_PYTHON: "false" UPGRADE_GOLANG: "true" UPGRADE_PIP: "false" - UPGRADE_PRE_COMMIT: "false" + UPGRADE_PREK: "false" UPGRADE_NODE_LTS: "false" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: "Run automated upgrade for pip, pre-commit and node (failing if needed)" + - name: "Run automated upgrade for important versions minus uv(failing if needed)" run: > - pre-commit run + prek --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual update-installers-and-pre-commit + --hook-stage manual upgrade-important-versions if: always() env: UPGRADE_UV: "false" UPGRADE_PYTHON: "true" UPGRADE_GOLANG: "false" UPGRADE_PIP: "true" - UPGRADE_PRE_COMMIT: "true" + UPGRADE_PREK: "true" UPGRADE_NODE_LTS: "true" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 69de65de65ae8..b1551028fd78f 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -122,7 +122,7 @@ jobs: arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} selected-providers-list-as-string: >- ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} - skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} + skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }} skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} source-head-repo: ${{ steps.source-run-info.outputs.head-repo }} source-head-ref: ${{ steps.source-run-info.outputs.head-ref }} @@ -168,8 +168,8 @@ jobs: PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} GITHUB_CONTEXT: ${{ toJson(github) }} - run-pin-versions-pre-commit: - name: "Run pin-versions pre-commit" + run-pin-versions-prek: + name: "Run pin-versions prek" needs: [build-info] runs-on: ${{ fromJSON(needs.build-info.outputs.amd-runners) }} steps: @@ -177,24 +177,22 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - - name: "Install Python 3.11 as 3.11+ is needed by pin-versions pre-commit" + - name: "Install Python 3.11 as 3.11+ is needed by pin-versions" uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 with: python-version: 3.11 cache: "pip" - - name: Install pre-commit, uv, and pre-commit-uv + - name: Install prek, uv shell: bash env: - UV_VERSION: "0.8.8" # Keep this comment to allow automatic replacement of uv version - PRE_COMMIT_VERSION: "4.3.0" # Keep this comment to allow automatic replacement of pre-commit version - PRE_COMMIT_UV_VERSION: "4.1.4" # Keep this comment to allow automatic replacement of pre-commit-uv version + UV_VERSION: "0.8.11" # Keep this comment to allow automatic replacement of uv version + PREK_VERSION: "0.0.26" # Keep this comment to allow automatic replacement of prek version run: | pip install uv==${UV_VERSION} || true - uv tool install pre-commit==${PRE_COMMIT_VERSION} --with uv==${UV_VERSION} \ - --with pre-commit-uv==${PRE_COMMIT_UV_VERSION} - - name: "Run pin-versions pre-commit" + uv tool install prek==${PREK_VERSION} --with uv==${UV_VERSION} + - name: "Run pin-versions" run: > - pre-commit run -c .github/.pre-commit-config.yaml --all-files --verbose --hook-stage manual + prek -c .github/.pre-commit-config.yaml --all-files --verbose --hook-stage manual pin-versions env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -210,7 +208,7 @@ jobs: needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} - skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} + skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}} canary-run: ${{needs.build-info.outputs.canary-run}} latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} use-uv: ${{needs.build-info.outputs.use-uv}} @@ -255,7 +253,7 @@ jobs: constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} canary-run: ${{ needs.build-info.outputs.canary-run }} @@ -301,7 +299,7 @@ jobs: latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index b17231cddf7d8..f7fbc9ae312b2 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -115,7 +115,7 @@ jobs: arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} selected-providers-list-as-string: >- ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} - skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} + skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }} skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} @@ -171,7 +171,7 @@ jobs: needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} - skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} + skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}} canary-run: ${{needs.build-info.outputs.canary-run}} latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} use-uv: ${{needs.build-info.outputs.use-uv}} @@ -216,7 +216,7 @@ jobs: constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} canary-run: ${{ needs.build-info.outputs.canary-run }} diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 7fd13495aa827..2d52aec1ad04f 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -72,8 +72,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to build CI images (true/false)" required: true type: string - skip-pre-commits: - description: "Whether to skip pre-commits (true/false)" + skip-prek-hooks: + description: "Whether to skip prek hooks (true/false)" required: true type: string include-success-outputs: @@ -122,9 +122,9 @@ on: # yamllint disable-line rule:truthy permissions: contents: read jobs: - install-pre-commit: + install-prek: timeout-minutes: 5 - name: "Install pre-commit for cache (only canary runs)" + name: "Install prek for cache (only canary runs)" runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -143,23 +143,23 @@ jobs: uses: ./.github/actions/breeze id: breeze if: inputs.canary-run == 'true' - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{steps.breeze.outputs.host-python-version}} if: inputs.canary-run == 'true' - - name: "Prepare .tar file from pre-commit cache" + - name: "Prepare .tar file from prek cache" run: | - tar -C ~ -czf /tmp/cache-pre-commit.tar.gz .cache/pre-commit .cache/uv + tar -C ~ -czf /tmp/cache-prek.tar.gz .cache/prek .cache/uv shell: bash if: inputs.canary-run == 'true' - - name: "Save pre-commit cache" + - name: "Save prek cache" uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468 with: # yamllint disable rule:line-length - key: cache-pre-commit-v4-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} - path: /tmp/cache-pre-commit.tar.gz + key: cache-prek-v5-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + path: /tmp/cache-prek.tar.gz if-no-files-found: 'error' retention-days: '2' if: inputs.canary-run == 'true' @@ -168,7 +168,7 @@ jobs: timeout-minutes: 45 name: "Static checks" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-pre-commit + needs: install-prek env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}" @@ -189,16 +189,16 @@ jobs: python: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} id: breeze - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{steps.breeze.outputs.host-python-version}} - name: "Static checks" - run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment + run: prek --all-files --show-diff-on-failure --color always env: VERBOSE: "false" - SKIP: ${{ inputs.skip-pre-commits }} + SKIP: ${{ inputs.skip-prek-hooks }} COLUMNS: "202" SKIP_GROUP_OUTPUT: "true" DEFAULT_BRANCH: ${{ inputs.branch }} @@ -208,7 +208,7 @@ jobs: timeout-minutes: 45 name: "MyPy checks" runs-on: ${{ fromJSON(inputs.runners) }} - needs: install-pre-commit + needs: install-prek if: inputs.needs-mypy == 'true' strategy: fail-fast: false @@ -232,13 +232,13 @@ jobs: python: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} id: breeze - - name: "Install pre-commit" - uses: ./.github/actions/install-pre-commit - id: pre-commit + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek with: python-version: ${{steps.breeze.outputs.host-python-version}} - name: "MyPy checks for ${{ matrix.mypy-check }}" - run: pre-commit run --color always --verbose --hook-stage manual "$MYPY_CHECK" --all-files + run: prek --color always --verbose --hook-stage manual "$MYPY_CHECK" --all-files env: VERBOSE: "false" COLUMNS: "202" diff --git a/.github/workflows/news-fragment.yml b/.github/workflows/news-fragment.yml index 04e308c306138..45efd3364ea6d 100644 --- a/.github/workflows/news-fragment.yml +++ b/.github/workflows/news-fragment.yml @@ -60,4 +60,4 @@ jobs: env: BASE_REF: ${{ github.base_ref }} run: > - uv run scripts/ci/pre_commit/significant_newsfragments_checker.py + uv run scripts/ci/prek/significant_newsfragments_checker.py diff --git a/.gitpod.yml b/.gitpod.yml index fa77c07eda214..2a724753de4ec 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -24,13 +24,13 @@ image: gitpod/workspace-python-3.11 tasks: - init: ./scripts/ci/install_breeze.sh - - name: Install pre-commit + - name: Install prek openMode: split-right command: | printf '%s\n' "export PIP_USER=no" >> "$HOME/.bashrc" source "$HOME/.bashrc" - pip install pre-commit - pre-commit install + pip install prek + prek install echo "for running integration test with breeze" # Ports to expose on workspace startup diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bcecd22edc4c7..798c2ca2ec2d4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ default_language_version: python: python3 node: 22.18.0 golang: 1.24.0 -minimum_pre_commit_version: '3.2.0' +minimum_prek_version: '0.0.26' exclude: ^.*/.*_vendor/ repos: - repo: meta @@ -176,7 +176,7 @@ repos: hooks: - id: check-min-python-version name: Check minimum Python version - entry: ./scripts/ci/pre_commit/check_min_python_version.py + entry: ./scripts/ci/prek/check_min_python_version.py language: python additional_dependencies: ['rich>=12.4.4'] require_serial: true @@ -189,34 +189,38 @@ repos: require_serial: true - id: update-black-version name: Update black versions everywhere (manual) - entry: ./scripts/ci/pre_commit/update_black_version.py + entry: ./scripts/ci/prek/update_black_version.py stages: ['manual'] language: python files: ^\.pre-commit-config\.yaml$ additional_dependencies: ['pyyaml>=6.0.2'] pass_filenames: false require_serial: true - - id: update-installers-and-pre-commit - name: Update installers and pre-commit to latest (manual) - entry: ./scripts/ci/pre_commit/update_installers_and_pre_commit.py + - id: upgrade-important-versions + name: Upgrade important versions (manual) + entry: ./scripts/ci/prek/upgrade_important_versions.py stages: ['manual'] language: python - files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/update_installers_and_pre_commit\.py$ + files: > + (?x) + ^\.pre-commit-config\.yaml$| + ^\.github/\.pre-commit-config\.yaml$| + ^scripts/ci/prek/update_installers_and_prek\.py$ pass_filenames: false require_serial: true - additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0',"packaging>=25"] + additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0',"packaging>=23.2"] - id: update-chart-dependencies name: Update chart dependencies to latest (manual) - entry: ./scripts/ci/pre_commit/update_chart_dependencies.py + entry: ./scripts/ci/prek/update_chart_dependencies.py stages: ['manual'] language: python additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4', 'requests>=2.31.0'] - files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/update_build_dependencies\.py$ + files: ^\.pre-commit-config\.yaml$|^scripts/ci/prek/update_build_dependencies\.py$ pass_filenames: false require_serial: true - id: check-taskinstance-tis-attrs name: Check that TI and TIS have the same attributes - entry: ./scripts/ci/pre_commit/check_ti_vs_tis_attributes.py + entry: ./scripts/ci/prek/check_ti_vs_tis_attributes.py language: python additional_dependencies: ['rich>=12.4.4'] files: ^airflow-core/src/airflow/models/taskinstance\.py$|^airflow-core/src/airflow/models/taskinstancehistory\.py$ @@ -225,7 +229,7 @@ repos: - id: check-deferrable-default name: Check and fix default value of default_deferrable language: python - entry: ./scripts/ci/pre_commit/check_deferrable_default.py + entry: ./scripts/ci/prek/check_deferrable_default.py pass_filenames: false # libcst doesn't have source wheels for all PY except PY3.12, excluding it # libcst 1.8.1 doesn't include typing-extensions which is needed for Python 3.9 @@ -363,16 +367,16 @@ repos: entry: zizmor - repo: local # Note that this is the 2nd "local" repo group in the .pre-commit-config.yaml file. This is because - # we try to minimise the number of passes that must happen in order to apply some of the changes - # done by pre-commits. Some of the pre-commits not only check for errors but also fix them. This means - # that output from an earlier pre-commit becomes input to another pre-commit. Splitting the local - # scripts of our and adding some other non-local pre-commit in-between allows us to handle such + # we try to minimize the number of passes that must happen to apply some of the changes + # done by prek-hooks. Some of the prek hooks not only check for errors but also fix them. This means + # that output from an earlier prek hook becomes input to another prek hook. Splitting the local + # scripts of our and adding some other non-local prek hook in-between allows us to handle such # changes quickly - especially when we want the early modifications from the first local group - # to be applied before the non-local pre-commits are run + # to be applied before the non-local prek hooks are run hooks: - id: update-providers-dependencies name: Update dependencies for providers - entry: ./scripts/ci/pre_commit/update_providers_dependencies.py + entry: ./scripts/ci/prek/update_providers_dependencies.py language: python always_run: true pass_filenames: false @@ -381,13 +385,13 @@ repos: name: No templated field logic checks in operator __init__ description: Prevent templated field logic checks in operators' __init__ language: python - entry: ./scripts/ci/pre_commit/validate_operators_init.py + entry: ./scripts/ci/prek/validate_operators_init.py pass_filenames: true files: ^providers/.*/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$ additional_dependencies: ['rich>=12.4.4'] - id: update-providers-build-files name: Update providers build files - entry: ./scripts/ci/pre_commit/update_providers_build_files.py + entry: ./scripts/ci/prek/update_providers_build_files.py language: python pass_filenames: true files: | @@ -414,7 +418,7 @@ repos: - id: ruff-format name: Run 'ruff format' description: "Run 'ruff format' for extremely fast Python formatting" - entry: ./scripts/ci/pre_commit/ruff_format.py + entry: ./scripts/ci/prek/ruff_format.py language: python types_or: [python, pyi] args: [] @@ -423,7 +427,7 @@ repos: exclude: ^airflow-core/tests/unit/dags/test_imports\.py$ - id: replace-bad-characters name: Replace bad characters - entry: ./scripts/ci/pre_commit/replace_bad_characters.py + entry: ./scripts/ci/prek/replace_bad_characters.py language: python types: [file, text] exclude: ^clients/gen/go\.sh$|^\.gitmodules$|.*/dist/.*|\.go$|/go\.(mod|sum)$ @@ -431,7 +435,7 @@ repos: - id: lint-dockerfile name: Lint Dockerfile language: python - entry: ./scripts/ci/pre_commit/lint_dockerfile.py + entry: ./scripts/ci/prek/lint_dockerfile.py files: Dockerfile.*$ pass_filenames: true require_serial: true @@ -441,7 +445,7 @@ repos: files: ^airflow-core/src/airflow/.*\.py$ require_serial: true exclude: ^airflow-core/src/airflow/kubernetes/ - entry: ./scripts/ci/pre_commit/check_airflow_imports.py + entry: ./scripts/ci/prek/check_airflow_imports.py --pattern '^airflow\.kubernetes' --message "You should only import kubernetes code from `airflow.providers.cncf.kubernetes`." additional_dependencies: ['rich>=12.4.4'] @@ -454,7 +458,7 @@ repos: (?x) ^airflow-core/src/airflow/openlineage/| ^airflow/providers/common/compat/openlineage/facet.py$ - entry: ./scripts/ci/pre_commit/check_airflow_imports.py + entry: ./scripts/ci/prek/check_airflow_imports.py --pattern '^openlineage\.client\.(facet|run)' --message "You should import from `airflow.providers.common.compat.openlineage.facet` instead." additional_dependencies: ['rich>=12.4.4'] @@ -463,7 +467,7 @@ repos: language: python files: ^\.github/ISSUE_TEMPLATE/3-airflow_providers_bug_report\.yml$ require_serial: true - entry: ./scripts/ci/pre_commit/check_airflow_bug_report_template.py + entry: ./scripts/ci/prek/check_airflow_bug_report_template.py additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2'] - id: check-cncf-k8s-only-for-executors name: Check cncf.kubernetes imports used for executors only @@ -478,13 +482,13 @@ repos: ^airflow-core/src/airflow/serialization/serialized_objects\.py$| ^airflow-core/src/airflow/serialization/serializers/kubernetes\.py$| ^airflow-core/src/airflow/utils/sqlalchemy\.py$ - entry: ./scripts/ci/pre_commit/check_airflow_imports.py + entry: ./scripts/ci/prek/check_airflow_imports.py --pattern '^airflow\.providers\.cncf\.kubernetes' --message "Only few k8s executors exceptions are allowed to use `airflow.providers.cncf.kubernetes`." additional_dependencies: ['rich>=12.4.4'] - id: update-local-yml-file name: Update mounts in the local yml file - entry: ./scripts/ci/pre_commit/local_yml_mounts.py + entry: ./scripts/ci/prek/local_yml_mounts.py language: python files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$ pass_filenames: false @@ -492,7 +496,7 @@ repos: - id: check-sql-dependency-common-data-structure name: Check dependency of SQL providers description: Check dependency of SQL Providers with common data structure - entry: ./scripts/ci/pre_commit/check_common_sql_dependency.py + entry: ./scripts/ci/prek/check_common_sql_dependency.py language: python files: ^providers/.*/src/airflow/providers/.*/hooks/.*\.py$ additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2', 'packaging>=23.2'] @@ -502,18 +506,18 @@ repos: language: python files: ^airflow-core/docs/extra-packages-ref\.rst$|^hatch_build\.py$ pass_filenames: false - entry: ./scripts/ci/pre_commit/check_extra_packages_ref.py + entry: ./scripts/ci/prek/check_extra_packages_ref.py additional_dependencies: ['rich>=12.4.4', 'hatchling==1.27.0', 'tabulate>=0.9.0'] - id: check-extras-order name: Check order of extras in Dockerfile - entry: ./scripts/ci/pre_commit/check_order_dockerfile_extras.py + entry: ./scripts/ci/prek/check_order_dockerfile_extras.py language: python files: ^Dockerfile$ pass_filenames: false additional_dependencies: ['rich>=12.4.4'] - id: generate-airflow-diagrams name: Generate airflow diagrams - entry: ./scripts/ci/pre_commit/generate_airflow_diagrams.py + entry: ./scripts/ci/prek/generate_airflow_diagrams.py language: python files: > (?x) @@ -523,7 +527,7 @@ repos: additional_dependencies: ['rich>=12.4.4', 'diagrams>=0.23.4'] - id: generate-volumes-for-sources name: Generate volumes for docker compose - entry: ./scripts/ci/pre_commit/generate_volumes_for_sources.py + entry: ./scripts/ci/prek/generate_volumes_for_sources.py language: python files: ^providers/.*/provider\.yaml$ pass_filenames: false @@ -531,19 +535,19 @@ repos: additional_dependencies: ['rich>=12.4.4'] - id: update-supported-versions name: Updates supported versions in documentation - entry: ./scripts/ci/pre_commit/supported_versions.py + entry: ./scripts/ci/prek/supported_versions.py language: python - files: ^airflow-core/docs/installation/supported-versions\.rst$|^scripts/ci/pre_commit/supported_versions\.py$|^README\.md$ + files: ^airflow-core/docs/installation/supported-versions\.rst$|^scripts/ci/prek/supported_versions\.py$|^README\.md$ pass_filenames: false additional_dependencies: ['tabulate>=0.9.0'] - id: check-revision-heads-map name: Check that the REVISION_HEADS_MAP is up-to-date language: python - entry: ./scripts/ci/pre_commit/check_revision_heads_map.py + entry: ./scripts/ci/prek/check_revision_heads_map.py pass_filenames: false files: > (?x) - ^scripts/ci/pre_commit/version_heads_map\.py$| + ^scripts/ci/prek/version_heads_map\.py$| ^airflow-core/src/airflow/migrations/versions/.*$|^airflow-core/src/airflow/migrations/versions| ^providers/fab/src/airflow/providers/fab/migrations/versions/.*$|^providers/fab/src/airflow/providers/fab/migrations/versions| ^airflow-core/src/airflow/utils/db\.py$| @@ -551,7 +555,7 @@ repos: additional_dependencies: ['packaging>=23.2', 'rich>=12.4.4'] - id: update-version name: Update versions in docs - entry: ./scripts/ci/pre_commit/update_versions.py + entry: ./scripts/ci/prek/update_versions.py language: python files: ^docs|^airflow-core/src/airflow/__init__\.py$|.*/pyproject\.toml$ pass_filenames: false @@ -603,7 +607,7 @@ repos: - id: check-only-new-session-with-provide-session name: Check NEW_SESSION is only used with @provide_session language: python - entry: ./scripts/ci/pre_commit/new_session_in_provide_session.py + entry: ./scripts/ci/prek/new_session_in_provide_session.py pass_filenames: true files: ^airflow-core/src/airflow/.+\.py$ exclude: ^airflow-core/src/airflow/serialization/pydantic/.* @@ -679,7 +683,7 @@ repos: ^.*/provider_conf\.py$| ^devel-common/src/sphinx_exts/removemarktransform\.py| ^airflow-core/newsfragments/41761.significant\.rst$| - ^scripts/ci/pre_commit/vendor_k8s_json_schema\.py$| + ^scripts/ci/prek/vendor_k8s_json_schema\.py$| ^scripts/ci/docker-compose/integration-keycloak\.yml$| ^scripts/ci/docker-compose/keycloak/keycloak-entrypoint\.sh$| ^airflow-core/tests/| @@ -699,21 +703,21 @@ repos: - id: check-base-operator-partial-arguments name: Check BaseOperator and partial() arguments language: python - entry: ./scripts/ci/pre_commit/check_base_operator_partial_arguments.py + entry: ./scripts/ci/prek/check_base_operator_partial_arguments.py pass_filenames: false files: ^airflow-core/src/airflow/models/(?:base|mapped)operator\.py$ additional_dependencies: ['rich>=12.4.4'] - id: check-init-decorator-arguments name: Sync model __init__ and decorator arguments language: python - entry: ./scripts/ci/pre_commit/check_init_decorator_arguments.py + entry: ./scripts/ci/prek/check_init_decorator_arguments.py pass_filenames: false files: ^task-sdk/src/airflow/sdk/definitions/dag\.py$|^task-sdk/src/airflow/sdk/definitions/decorators/task_group\.py$ additional_dependencies: ['rich>=12.4.4'] - id: check-template-context-variable-in-sync name: Sync template context variable refs language: python - entry: ./scripts/ci/pre_commit/check_template_context_variable_in_sync.py + entry: ./scripts/ci/prek/check_template_context_variable_in_sync.py files: ^airflow-core/src/airflow/models/taskinstance\.py$|^task-sdk/src/airflow/sdk/definitions/context\.py$|^airflow-core/docs/templates-ref\.rst$ additional_dependencies: ['rich>=12.4.4'] - id: check-base-operator-usage @@ -758,7 +762,7 @@ repos: name: Check providers import hook lineage code from compat description: Make sure you import from airflow.provider.common.compat.lineage.hook instead of airflow.lineage.hook. - entry: ./scripts/ci/pre_commit/check_airflow_imports.py + entry: ./scripts/ci/prek/check_airflow_imports.py --pattern '^airflow\.lineage\.hook' --message "Only TYPE_CHECKING imports from `airflow.lineage.hook` are allowed in providers." --only_top_level @@ -768,7 +772,7 @@ repos: - id: check-decorated-operator-implements-custom-name name: Check @task decorator implements custom_operator_name language: python - entry: ./scripts/ci/pre_commit/decorator_operator_implements_custom_name.py + entry: ./scripts/ci/prek/decorator_operator_implements_custom_name.py pass_filenames: true files: ^airflow-core/src/airflow/.*\.py$ - id: check-core-deprecation-classes @@ -805,13 +809,13 @@ repos: pass_filenames: true - id: check-apache-license-rat name: Check if licenses are OK for Apache - entry: ./scripts/ci/pre_commit/check_license.py + entry: ./scripts/ci/prek/check_license.py language: python files: ^LICENSE$ pass_filenames: false - id: check-aiobotocore-optional name: Check if aiobotocore is an optional dependency only - entry: ./scripts/ci/pre_commit/check_aiobotocore_optional.py + entry: ./scripts/ci/prek/check_aiobotocore_optional.py language: python files: ^providers/.*/provider\.yaml$ pass_filenames: true @@ -820,34 +824,34 @@ repos: - id: check-boring-cyborg-configuration name: Checks for Boring Cyborg configuration consistency language: python - entry: ./scripts/ci/pre_commit/boring_cyborg.py + entry: ./scripts/ci/prek/boring_cyborg.py pass_filenames: false require_serial: true additional_dependencies: ['pyyaml>=6.0.2', 'termcolor==2.5.0', 'wcmatch==8.2'] - id: update-in-the-wild-to-be-sorted name: Sort INTHEWILD.md alphabetically - entry: ./scripts/ci/pre_commit/sort_in_the_wild.py + entry: ./scripts/ci/prek/sort_in_the_wild.py language: python files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$ pass_filenames: false require_serial: true - id: update-installed-providers-to-be-sorted name: Sort and uniquify installed_providers.txt - entry: ./scripts/ci/pre_commit/sort_installed_providers.py + entry: ./scripts/ci/prek/sort_installed_providers.py language: python files: ^\.pre-commit-config\.yaml$|^.*_installed_providers\.txt$ pass_filenames: false require_serial: true - id: update-spelling-wordlist-to-be-sorted name: Sort spelling_wordlist.txt - entry: ./scripts/ci/pre_commit/sort_spelling_wordlist.py + entry: ./scripts/ci/prek/sort_spelling_wordlist.py language: python files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$ require_serial: true pass_filenames: false - id: lint-helm-chart name: Lint Helm Chart - entry: ./scripts/ci/pre_commit/lint_helm.py + entry: ./scripts/ci/prek/lint_helm.py language: python pass_filenames: false files: ^chart @@ -855,7 +859,7 @@ repos: additional_dependencies: ['rich>=12.4.4','requests>=2.31.0'] - id: kubeconform name: Kubeconform check on our helm chart - entry: ./scripts/ci/pre_commit/check_kubeconform.py + entry: ./scripts/ci/prek/check_kubeconform.py language: python pass_filenames: false files: ^chart @@ -873,14 +877,14 @@ repos: stages: ['manual'] types_or: [javascript, ts, tsx] files: ^airflow-core/src/airflow/ui/|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/ - entry: ./scripts/ci/pre_commit/compile_ui_assets.py + entry: ./scripts/ci/prek/compile_ui_assets.py pass_filenames: false additional_dependencies: ['pnpm@9.7.1'] - id: compile-fab-assets name: Compile FAB provider assets language: node files: ^providers/fab/.*/www/ - entry: ./scripts/ci/pre_commit/compile_fab_assets.py + entry: ./scripts/ci/prek/compile_fab_assets.py pass_filenames: false additional_dependencies: ['yarn@1.22.21'] - id: compile-ui-assets-dev @@ -889,30 +893,20 @@ repos: stages: ['manual'] types_or: [javascript, ts, tsx] files: ^airflow-core/src/airflow/ui/|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/ - entry: ./scripts/ci/pre_commit/compile_ui_assets_dev.py + entry: ./scripts/ci/prek/compile_ui_assets_dev.py pass_filenames: false additional_dependencies: ['pnpm@9.7.1'] - id: check-providers-subpackages-init-file-exist name: Provider subpackage init files are there pass_filenames: false always_run: true - entry: ./scripts/ci/pre_commit/check_providers_subpackages_all_have_init.py + entry: ./scripts/ci/prek/check_providers_subpackages_all_have_init.py language: python require_serial: true additional_dependencies: ['rich>=12.4.4'] - - id: check-pre-commit-information-consistent - name: Validate hook IDs & names and sync with docs - entry: ./scripts/ci/pre_commit/check_pre_commit_hooks.py - args: - - --max-length=53 - language: python - files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/check_pre_commit_hooks\.py$ - additional_dependencies: ['pyyaml>=6.0.2', 'jinja2', 'black==24.10.0', 'tabulate>=0.9.0', 'rich>=12.4.4'] - require_serial: true - pass_filenames: false - id: check-integrations-list-consistent name: Sync integrations list with docs - entry: ./scripts/ci/pre_commit/check_integrations_list.py + entry: ./scripts/ci/prek/check_integrations_list.py language: python files: ^scripts/ci/docker-compose/integration-.*\.yml$|^contributing-docs/testing/integration_tests\.rst$ additional_dependencies: ['black==24.10.0', 'tabulate>=0.9.0', 'rich>=12.4.4', 'pyyaml>=6.0.2'] @@ -921,7 +915,7 @@ repos: - id: update-breeze-readme-config-hash name: Update Breeze README.md with config files hash language: python - entry: ./scripts/ci/pre_commit/update_breeze_config_hash.py + entry: ./scripts/ci/prek/update_breeze_config_hash.py files: ^dev/breeze/pyproject\.toml$|^dev/breeze/README\.md$ pass_filenames: false require_serial: true @@ -929,15 +923,20 @@ repos: - id: update-pyproject-toml name: Update Airflow's meta-package pyproject.toml language: python - entry: ./scripts/ci/pre_commit/update_airflow_pyproject_toml.py - files: ^.*/pyproject\.toml$|^scripts/ci/pre_commit/update_airflow_pyproject_toml\.py$ + entry: ./scripts/ci/prek/update_airflow_pyproject_toml.py + files: > + (?x) + ^.*/pyproject\.toml$| + ^scripts/ci/prek/update_airflow_pyproject_toml\.py$| + ^providers/.*/pyproject\.toml$| + ^providers/.*/provider\.yaml$ pass_filenames: false require_serial: true additional_dependencies: ['rich>=12.4.4', 'tomli>=2.0.1', 'packaging>=23.2' ] - id: update-reproducible-source-date-epoch name: Update Source Date Epoch for reproducible builds language: python - entry: ./scripts/ci/pre_commit/update_source_date_epoch.py + entry: ./scripts/ci/prek/update_source_date_epoch.py files: ^RELEASE_NOTES\.rst$|^chart/RELEASE_NOTES\.rst$ additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2'] pass_filenames: false @@ -953,7 +952,7 @@ repos: additional_dependencies: ['click>=8.1.8', 'rich>=12.4.4', 'pyyaml>=6.0.2'] - id: check-tests-in-the-right-folders name: Check if tests are in the right folders - entry: ./scripts/ci/pre_commit/check_tests_in_right_folders.py + entry: ./scripts/ci/prek/check_tests_in_right_folders.py language: python files: ^airflow-core/tests/.*\.py$ pass_filenames: true @@ -961,14 +960,14 @@ repos: additional_dependencies: ['rich>=12.4.4'] - id: check-system-tests-present name: Check if system tests have required segments of code - entry: ./scripts/ci/pre_commit/check_system_tests.py + entry: ./scripts/ci/prek/check_system_tests.py language: python files: ^.*/tests/system/.*/example_[^/]*\.py$ pass_filenames: true additional_dependencies: ['rich>=12.4.4'] - id: generate-pypi-readme name: Generate PyPI README - entry: ./scripts/ci/pre_commit/generate_pypi_readme.py + entry: ./scripts/ci/prek/generate_pypi_readme.py language: python files: ^README\.md$ pass_filenames: false @@ -982,10 +981,10 @@ repos: additional_dependencies: ['markdownlint-cli@0.38.0'] - id: lint-json-schema name: Lint JSON Schema files - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-file - - scripts/ci/pre_commit/draft7_schema.json + - scripts/ci/prek/draft7_schema.json language: python pass_filenames: true files: .*\.schema\.json$ @@ -993,7 +992,7 @@ repos: additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] - id: lint-json-schema name: Lint NodePort Service - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-url - https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.20.2-standalone/service-v1.json @@ -1004,7 +1003,7 @@ repos: additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] - id: lint-json-schema name: Lint Docker compose files - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-url - https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json @@ -1020,7 +1019,7 @@ repos: additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] - id: lint-json-schema name: Lint chart/values.schema.json - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-file - chart/values_schema.schema.json @@ -1032,13 +1031,13 @@ repos: additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] - id: update-vendored-in-k8s-json-schema name: Vendor k8s definitions into values.schema.json - entry: ./scripts/ci/pre_commit/vendor_k8s_json_schema.py + entry: ./scripts/ci/prek/vendor_k8s_json_schema.py language: python files: ^chart/values\.schema\.json$ additional_dependencies: ['requests==2.32.3'] - id: lint-json-schema name: Lint chart/values.yaml - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --enforce-defaults - --spec-file @@ -1051,7 +1050,7 @@ repos: additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] - id: lint-json-schema name: Lint config_templates/config.yml - entry: ./scripts/ci/pre_commit/lint_json_schema.py + entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-file - airflow-core/src/airflow/config_templates/config.yml.schema.json @@ -1063,14 +1062,14 @@ repos: - id: check-persist-credentials-disabled-in-github-workflows name: Check persistent creds in workflow files description: Check that workflow files have persist-credentials disabled - entry: ./scripts/ci/pre_commit/checkout_no_credentials.py + entry: ./scripts/ci/prek/checkout_no_credentials.py language: python pass_filenames: true files: ^\.github/workflows/.*\.yml$ additional_dependencies: ['pyyaml>=6.0.1', 'rich>=12.4.4'] - id: check-docstring-param-types name: Check that docstrings do not specify param types - entry: ./scripts/ci/pre_commit/docstring_param_type.py + entry: ./scripts/ci/prek/docstring_param_type.py language: python pass_filenames: true files: \.py$ @@ -1085,21 +1084,21 @@ repos: files: \.zip$ - id: check-code-deprecations name: Check deprecations categories in decorators - entry: ./scripts/ci/pre_commit/check_deprecations.py + entry: ./scripts/ci/prek/check_deprecations.py language: python pass_filenames: true files: ^airflow-core/src/airflow/.*\.py$ additional_dependencies: ['rich>=12.4.4', 'python-dateutil>=2.9.0'] - id: lint-chart-schema name: Lint chart/values.schema.json file - entry: ./scripts/ci/pre_commit/chart_schema.py + entry: ./scripts/ci/prek/chart_schema.py language: python pass_filenames: false files: ^chart/values\.schema\.json$ require_serial: true - id: update-inlined-dockerfile-scripts name: Inline Dockerfile and Dockerfile.ci scripts - entry: ./scripts/ci/pre_commit/inline_scripts_in_docker.py + entry: ./scripts/ci/prek/inline_scripts_in_docker.py language: python pass_filenames: false files: ^Dockerfile$|^Dockerfile\.ci$|^scripts/docker/.*$ @@ -1108,19 +1107,19 @@ repos: name: Check changelogs for duplicate entries language: python files: changelog\.(rst|txt)$ - entry: ./scripts/ci/pre_commit/changelog_duplicates.py + entry: ./scripts/ci/prek/changelog_duplicates.py pass_filenames: true - id: check-changelog-format name: Check changelog format language: python files: changelog\.(rst|txt)$ - entry: ./scripts/ci/pre_commit/check_changelog_format.py + entry: ./scripts/ci/prek/check_changelog_format.py pass_filenames: true - id: check-newsfragments-are-valid name: Check newsfragments are valid language: python files: newsfragments/.*\.rst$ - entry: ./scripts/ci/pre_commit/newsfragments.py + entry: ./scripts/ci/prek/newsfragments.py pass_filenames: true # We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes # This is fast, so not too much downside @@ -1129,8 +1128,8 @@ repos: name: Check significant newsfragments are valid # Significant newsfragments follows a special format so that we can group information easily. language: python - files: airflow-core/newsfragments/.*\.rst$ - entry: ./scripts/ci/pre_commit/significant_newsfragments_checker.py + files: ^airflow-core/newsfragments/.*\.rst$ + entry: ./scripts/ci/prek/significant_newsfragments_checker.py pass_filenames: false additional_dependencies: ['docutils>=0.21.2', 'pygments>=2.19.1', 'jinja2>=3.1.5'] # We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes @@ -1139,20 +1138,20 @@ repos: - id: update-breeze-cmd-output name: Update breeze docs description: Update output of breeze commands in Breeze documentation - entry: ./scripts/ci/pre_commit/breeze_cmd_line.py + entry: ./scripts/ci/prek/breeze_cmd_line.py language: python files: > (?x) ^dev/breeze/.*$| ^\.pre-commit-config\.yaml$| - ^scripts/ci/pre_commit/breeze_cmd_line\.py$| + ^scripts/ci/prek/breeze_cmd_line\.py$| ^generated/provider_dependencies\.json$ require_serial: true pass_filenames: false additional_dependencies: ['rich>=12.4.4'] - id: check-example-dags-urls name: Check that example dags url include provider versions - entry: ./scripts/ci/pre_commit/update_example_dags_paths.py + entry: ./scripts/ci/prek/update_example_dags_paths.py language: python pass_filenames: true files: ^airflow-core/docs/.*example-dags\.rst$|^docs/.*index\.rst$^airflow-core/docs/.*index\.rst$ @@ -1160,21 +1159,21 @@ repos: always_run: true - id: check-system-tests-tocs name: Check that system tests is properly added - entry: ./scripts/ci/pre_commit/check_system_tests_hidden_in_index.py + entry: ./scripts/ci/prek/check_system_tests_hidden_in_index.py language: python pass_filenames: true files: ^providers/.*/docs/index\.rst$ additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2'] - id: check-lazy-logging name: Check that all logging methods are lazy - entry: ./scripts/ci/pre_commit/check_lazy_logging.py + entry: ./scripts/ci/prek/check_lazy_logging.py language: python pass_filenames: true files: \.py$ additional_dependencies: ['rich>=12.4.4', 'astor>=0.8.1'] - id: create-missing-init-py-files-tests name: Create missing init.py files in tests - entry: ./scripts/ci/pre_commit/check_init_in_tests.py + entry: ./scripts/ci/prek/check_init_in_tests.py language: python additional_dependencies: ['rich>=12.4.4'] pass_filenames: false @@ -1182,13 +1181,13 @@ repos: - id: check-tests-unittest-testcase name: Unit tests do not inherit from unittest.TestCase description: Check that unit tests do not inherit from unittest.TestCase - entry: ./scripts/ci/pre_commit/unittest_testcase.py + entry: ./scripts/ci/prek/unittest_testcase.py language: python pass_filenames: true files: ^airflow-core/tests/.*\.py$ - id: check-provider-docs-valid name: Validate provider doc files - entry: ./scripts/ci/pre_commit/check_provider_docs.py + entry: ./scripts/ci/prek/check_provider_docs.py language: python files: ^providers/.*/provider\.yaml$|^.*/docs/.* additional_dependencies: ['rich>=12.4.4', 'pyyaml>=6.0.2', 'jinja2'] @@ -1266,7 +1265,7 @@ repos: (?x) ^airflow-core/src/airflow/ui/node-modules/.*| ^airflow-core/src/airflow/ui/.pnpm-store - entry: ./scripts/ci/pre_commit/ts_compile_lint_ui.py + entry: ./scripts/ci/prek/ts_compile_lint_ui.py additional_dependencies: ['pnpm@9.7.1'] pass_filenames: true require_serial: true @@ -1283,16 +1282,16 @@ repos: (?x) ^airflow-core/src/airflow/api_fastapi/node-modules/.*| ^airflow-core/src/airflow/api_fastapi/.pnpm-store - entry: ./scripts/ci/pre_commit/ts_compile_lint_simple_auth_manager_ui.py + entry: ./scripts/ci/prek/ts_compile_lint_simple_auth_manager_ui.py additional_dependencies: ['pnpm@9.7.1'] pass_filenames: true require_serial: true - ## ADD MOST PRE-COMMITS ABOVE THAT LINE - # The below pre-commits are those requiring CI image to be built + ## ADD MOST PREK HOOK ABOVE THAT LINE + # The below prek hooks are those requiring CI image to be built - id: mypy-dev name: Run mypy for dev language: python - entry: ./scripts/ci/pre_commit/mypy.py + entry: ./scripts/ci/prek/mypy.py files: ^dev/.*\.py$|^scripts/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] @@ -1300,7 +1299,7 @@ repos: stages: ['manual'] name: Run mypy for dev (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py dev + entry: ./scripts/ci/prek/mypy_folder.py dev pass_filenames: false files: ^.*\.py$ require_serial: true @@ -1308,15 +1307,15 @@ repos: - id: mypy-airflow-core name: Run mypy for airflow-core language: python - entry: ./scripts/ci/pre_commit/mypy.py - files: airflow-core/.*\.py$ + entry: ./scripts/ci/prek/mypy.py + files: ^airflow-core/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] - id: mypy-airflow-core stages: ['manual'] name: Run mypy for airflow-core (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py airflow-core + entry: ./scripts/ci/prek/mypy_folder.py airflow-core pass_filenames: false files: airflow-core/.*\.py$ require_serial: true @@ -1324,7 +1323,7 @@ repos: - id: mypy-providers name: Run mypy for providers language: python - entry: ./scripts/ci/pre_commit/mypy.py + entry: ./scripts/ci/prek/mypy.py files: ^providers/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] @@ -1332,7 +1331,7 @@ repos: stages: ['manual'] name: Run mypy for providers (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py providers + entry: ./scripts/ci/prek/mypy_folder.py providers pass_filenames: false files: ^.*\.py$ require_serial: true @@ -1340,7 +1339,7 @@ repos: - id: mypy-task-sdk name: Run mypy for task-sdk language: python - entry: ./scripts/ci/pre_commit/mypy.py + entry: ./scripts/ci/prek/mypy.py files: ^task-sdk/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] @@ -1348,7 +1347,7 @@ repos: stages: ['manual'] name: Run mypy for task-sdk (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py task-sdk + entry: ./scripts/ci/prek/mypy_folder.py task-sdk pass_filenames: false files: ^.*\.py$ require_serial: true @@ -1356,7 +1355,7 @@ repos: - id: mypy-devel-common name: Run mypy for devel-common language: python - entry: ./scripts/ci/pre_commit/mypy.py + entry: ./scripts/ci/prek/mypy.py files: ^devel-common/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] @@ -1364,24 +1363,7 @@ repos: stages: ['manual'] name: Run mypy for devel-common (manual) language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py devel-common - pass_filenames: false - files: ^.*\.py$ - require_serial: true - additional_dependencies: ['rich>=12.4.4'] - - id: mypy-airflow-ctl - name: Run mypy for airflow-ctl - language: python - entry: ./scripts/ci/pre_commit/mypy.py - files: ^airflow-ctl/src/airflowctl/.*\.py$|^airflow-ctl/tests/.*\.py$ - exclude: .*generated.py - require_serial: true - additional_dependencies: ['rich>=12.4.4'] - - id: mypy-airflow-ctl - stages: ['manual'] - name: Run mypy for airflow-ctl (manual) - language: python - entry: ./scripts/ci/pre_commit/mypy_folder.py airflow-ctl + entry: ./scripts/ci/prek/mypy_folder.py devel-common pass_filenames: false files: ^.*\.py$ require_serial: true @@ -1389,7 +1371,7 @@ repos: - id: generate-openapi-spec name: Generate the FastAPI API spec language: python - entry: ./scripts/ci/pre_commit/generate_openapi_spec.py + entry: ./scripts/ci/prek/generate_openapi_spec.py pass_filenames: false files: ^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$|^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ exclude: ^airflow-core/src/airflow/api_fastapi/execution_api/.* @@ -1397,14 +1379,14 @@ repos: - id: generate-openapi-spec-fab name: Generate the FastAPI API spec for FAB language: python - entry: ./scripts/ci/pre_commit/generate_openapi_spec_fab.py + entry: ./scripts/ci/prek/generate_openapi_spec_fab.py pass_filenames: false files: ^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$|^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ exclude: ^airflow-core/src/airflow/api_fastapi/execution_api/.* additional_dependencies: ['rich>=12.4.4', 'openapi-spec-validator>=0.7.1'] - id: check-provider-yaml-valid name: Validate provider.yaml files - entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py + entry: ./scripts/ci/prek/check_provider_yaml_files.py language: python files: ^providers/.*/provider\.yaml$ exclude: ^providers/.*/.venv/.*$ @@ -1413,14 +1395,14 @@ repos: - id: check-template-fields-valid name: Check templated fields mapped in operators/sensors language: python - entry: ./scripts/ci/pre_commit/check_template_fields.py + entry: ./scripts/ci/prek/check_template_fields.py files: ^(providers/.*/)?airflow-core/.*/(sensors|operators)/.*\.py$ additional_dependencies: ['rich>=12.4.4'] require_serial: true - id: update-migration-references name: Update migration ref doc language: python - entry: ./scripts/ci/pre_commit/migration_reference.py + entry: ./scripts/ci/prek/migration_reference.py pass_filenames: false files: ^airflow-core/src/airflow/migrations/versions/.*\.py$|^airflow-core/docs/migrations-ref\.rst$ additional_dependencies: ['rich>=12.4.4'] @@ -1434,16 +1416,16 @@ repos: - id: update-er-diagram name: Update ER diagram language: python - entry: ./scripts/ci/pre_commit/update_er_diagram.py + entry: ./scripts/ci/prek/update_er_diagram.py pass_filenames: false files: ^airflow-core/src/airflow/migrations/versions/.*\.py$|^airflow-core/docs/migrations-ref\.rst$ additional_dependencies: ['rich>=12.4.4'] - id: check-default-configuration name: Check the default configuration - entry: ./scripts/ci/pre_commit/check_default_configuration.py + entry: ./scripts/ci/prek/check_default_configuration.py language: python additional_dependencies: ['rich>=12.4.4'] require_serial: true pass_filenames: false files: ^airflow-core/src/airflow/config_templates/config\.yml$ - ## ONLY ADD PRE-COMMITS HERE THAT REQUIRE CI IMAGE + ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE diff --git a/.rat-excludes b/.rat-excludes index 95e31f35fba37..1c2a620f8f6ae 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -175,3 +175,11 @@ auth_generated.py # hash files www-hash.txt + +# go setup files +go.mod +go.sum +mocks/* + +# Kubernetes env +.env diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000000..43afab4e7db41 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,47 @@ + + +# AGENTS instructions + +The main developer documentation lives in the `contributing-docs` directory. The following points summarise +how to set up the environment, run checks, build docs and follow the PR workflow. + +## Local virtualenv and Breeze + +- [`07_local_virtualenv.rst`](contributing-docs/07_local_virtualenv.rst) explains how to prepare a local Python environment using `uv`. The tool creates and syncs a `.venv` and installs dependencies with commands such as `uv venv` and `uv sync`. +- [`06_development_environments.rst`](contributing-docs/06_development_environments.rst) compares the local virtualenv with the Docker based Breeze environment. Breeze replicates CI and includes services like databases for integration tests. + +## Prek hooks + +- Installation and usage of `prek` are described in [`03_contributors_quick_start.rst`](contributing-docs/03_contributors_quick_start.rst). Install with `uv tool install prek` and run checks via `prek --all-files`. +- [`08_static_code_checks.rst`](contributing-docs/08_static_code_checks.rst) provides more details on the available hooks and prerequisites. Enable the hooks with `prek install` so they run automatically on each commit. + +## Running tests + +- [`03_contributors_quick_start.rst`](contributing-docs/03_contributors_quick_start.rst) shows running tests inside Breeze. Use `pytest` inside the container for individual files or invoke `breeze testing` commands to run full suites, e.g. `breeze --backend postgres --python 3.10 testing tests --test-type All`. + +## Building documentation + +- Documentation can be built locally using `uv run --group docs build-docs` as described in [`11_documentation_building.rst`](contributing-docs/11_documentation_building.rst). Within Breeze the equivalent command is `breeze build-docs`. + +## Pull request guidelines + +- Follow the PR guidance in [`05_pull_requests.rst`](contributing-docs/05_pull_requests.rst). Always add tests, keep your branch rebased instead of merged, and adhere to the commit message recommendations from [cbea.ms/git-commit](https://cbea.ms/git-commit/). + +For advanced topics such as packaging providers and API versioning see [`12_provider_distributions.rst`](contributing-docs/12_provider_distributions.rst) and [`19_execution_api_versioning.rst`](contributing-docs/19_execution_api_versioning.rst). diff --git a/Dockerfile b/Dockerfile index a272f59400254..f2cb78c15c62d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,7 +57,7 @@ ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" ARG AIRFLOW_PIP_VERSION=25.2 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" ARG AIRFLOW_SETUPTOOLS_VERSION=80.9.0 -ARG AIRFLOW_UV_VERSION=0.8.8 +ARG AIRFLOW_UV_VERSION=0.8.11 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" @@ -87,7 +87,7 @@ FROM scratch as scripts ############################################################################################## # Please DO NOT modify the inlined scripts manually. The content of those files will be -# replaced by pre-commit automatically from the "scripts/docker/" folder. +# replaced by prek automatically from the "scripts/docker/" folder. # This is done in order to avoid problems with caching and file permissions and in order to # make the PROD Dockerfile standalone ############################################################################################## @@ -615,20 +615,19 @@ function common::install_packaging_tools() { pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" fi fi - if [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then + if [[ ${AIRFLOW_PREK_VERSION=} == "" ]]; then echo - echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv uv${COLOR_RESET}" + echo "${COLOR_BLUE}Installing latest prek, uv${COLOR_RESET}" echo - uv tool install pre-commit --with pre-commit-uv --with uv + uv tool install prek --with uv # make sure that the venv/user in .local exists mkdir -p "${HOME}/.local/bin" else echo - echo "${COLOR_BLUE}Installing predefined versions of pre-commit with pre-commit-uv and uv:${COLOR_RESET}" - echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION}) uv(${AIRFLOW_UV_VERSION}) pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}" + echo "${COLOR_BLUE}Installing predefined versions of prek, uv:${COLOR_RESET}" + echo "${COLOR_BLUE}prek(${AIRFLOW_PREK_VERSION}) uv(${AIRFLOW_UV_VERSION})${COLOR_RESET}" echo - uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \ - --with "uv==${AIRFLOW_UV_VERSION}" --with "pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}" + uv tool install "prek==${AIRFLOW_PREK_VERSION}" --with "uv==${AIRFLOW_UV_VERSION}" # make sure that the venv/user in .local exists mkdir -p "${HOME}/.local/bin" fi diff --git a/Dockerfile.ci b/Dockerfile.ci index bda99440e9efc..202a3eaf7dc31 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -26,7 +26,7 @@ FROM ${BASE_IMAGE} as scripts ############################################################################################## # Please DO NOT modify the inlined scripts manually. The content of those files will be -# replaced by pre-commit automatically from the "scripts/docker/" folder. +# replaced by prek automatically from the "scripts/docker/" folder. # This is done in order to avoid problems with caching and file permissions and in order to # make the PROD Dockerfile standalone ############################################################################################## @@ -584,20 +584,19 @@ function common::install_packaging_tools() { pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" fi fi - if [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then + if [[ ${AIRFLOW_PREK_VERSION=} == "" ]]; then echo - echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv uv${COLOR_RESET}" + echo "${COLOR_BLUE}Installing latest prek, uv${COLOR_RESET}" echo - uv tool install pre-commit --with pre-commit-uv --with uv + uv tool install prek --with uv # make sure that the venv/user in .local exists mkdir -p "${HOME}/.local/bin" else echo - echo "${COLOR_BLUE}Installing predefined versions of pre-commit with pre-commit-uv and uv:${COLOR_RESET}" - echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION}) uv(${AIRFLOW_UV_VERSION}) pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}" + echo "${COLOR_BLUE}Installing predefined versions of prek, uv:${COLOR_RESET}" + echo "${COLOR_BLUE}prek(${AIRFLOW_PREK_VERSION}) uv(${AIRFLOW_UV_VERSION})${COLOR_RESET}" echo - uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \ - --with "uv==${AIRFLOW_UV_VERSION}" --with "pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}" + uv tool install "prek==${AIRFLOW_PREK_VERSION}" --with "uv==${AIRFLOW_UV_VERSION}" # make sure that the venv/user in .local exists mkdir -p "${HOME}/.local/bin" fi @@ -1314,7 +1313,7 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ARG AIRFLOW_PYTHON_VERSION=v3.10.18 ENV AIRFLOW_PYTHON_VERSION=$AIRFLOW_PYTHON_VERSION -ENV GOLANG_MAJOR_MINOR_VERSION=1.24.6 +ENV GOLANG_MAJOR_MINOR_VERSION=1.25.0 COPY --from=scripts install_os_dependencies_ci.sh /scripts/docker/ @@ -1445,17 +1444,16 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe ARG AIRFLOW_PIP_VERSION=25.2 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" ARG AIRFLOW_SETUPTOOLS_VERSION=80.9.0 -ARG AIRFLOW_UV_VERSION=0.8.8 +ARG AIRFLOW_UV_VERSION=0.8.11 # TODO(potiuk): automate with upgrade check (possibly) -ARG AIRFLOW_PRE_COMMIT_VERSION="4.3.0" -ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4" +ARG AIRFLOW_PREK_VERSION="0.0.26" ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_SETUPTOOLS_VERSION=${AIRFLOW_SETUPTOOLS_VERSION} \ AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ # This is needed since we are using cache mounted from the host UV_LINK_MODE=copy \ - AIRFLOW_PRE_COMMIT_VERSION=${AIRFLOW_PRE_COMMIT_VERSION} + AIRFLOW_PREK_VERSION=${AIRFLOW_PREK_VERSION} # The PATH is needed for PIPX to find the tools installed ENV PATH="/root/.local/bin:${PATH}" diff --git a/INSTALL b/INSTALL index b7a6a51602623..07003c6693cde 100644 --- a/INSTALL +++ b/INSTALL @@ -80,12 +80,12 @@ by running the following command: uv sync --all-packages With `uv` you can also install tools that are needed for other tasks described later - breeze, -pre-commit, hatch, cherry-picker etc. It is highly recommended to install breeze and pre-commit, hatch +prek, hatch, cherry-picker etc. It is highly recommended to install breeze and prek, hatch and flit are useful to build packages (so might be useful by release managers), and cherry-picker is useful for backporting changes to previous versions of Airflow. uv tool install -e ./dev/breeze - uv tool install pre-commit --with pre-commit-uv + uv tool install prek uv tool install hatch uv tool install flit @@ -94,10 +94,10 @@ useful for backporting changes to previous versions of Airflow. Those are all tools useful for Airflow development. -It is recommended to run `pre-commit install` after installing `pre-commit` to install the git hooks - they -will take care about running airflow pre-commit checks automatically. +It is recommended to run `prek install` after installing `prek` to install the git hooks - they +will take care about running airflow prek checks automatically. - pre-commit install + prek install You can run any command in the virtual environment created by `uv` by prefixing it with `uv run`: @@ -112,16 +112,16 @@ Compiling front-end assets In order to see UI in Airflow, you need to compile front-end assets first. -In case you already installed `breeze` and `pre-commit`, you can build the assets with +In case you already installed `breeze` and `prek`, you can build the assets with the following commands: - pre-commit run --hook-stage manual compile-ui-assets --all-files + prek --hook-stage manual compile-ui-assets --all-files or simply: breeze compile-ui-assets -Both commands will install node and pmpm under the hood, to a dedicated pre-commit +Both commands will install node and pmpm under the hood, to a dedicated prek node environment and then build the assets. If you want to manually run the build check for node and pnpm version in `.pre-commit-config.yaml` file, @@ -140,7 +140,7 @@ This will also clean-up the generated files,update `git_version` file in the Air it in the UI) that should contain the git commit hash of the build and it will generate the assets. The result of this command is airflow sdist package built in the `dist` folder of `airflow-core` -package as well. It requires ``pre-commit`` to be installed in your system. +package as well. It requires ``prek`` to be installed in your system. Using pip and manually managing your virtualenv =============================================== diff --git a/PROVIDERS.rst b/PROVIDERS.rst index ced8f5fb1b4cd..1c8e613eb0fdd 100644 --- a/PROVIDERS.rst +++ b/PROVIDERS.rst @@ -344,5 +344,5 @@ The dependencies for Airflow providers are managed in the ``provider.yaml`` file All provider dependencies, including versions and constraints, are listed in this file. When adding or updating a provider or its dependencies, changes should be made to this file accordingly. -To ensure consistency and manage dependencies, ``pre-commit`` is configured to automatically update all dependencies. -Once you have ``pre-commit`` installed, it will automatically handle the dependency updates. +To ensure consistency and manage dependencies, ``prek`` is configured to automatically update all dependencies. +Once you have ``prek`` installed, it will automatically handle the dependency updates. diff --git a/README.md b/README.md index f0ea6a1986b03..0ba46bb06b8bf 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ packages: Apache Airflow version life cycle: - + | Version | Current Patch/Minor | State | First Release | Limited Maintenance | EOL/Terminated | diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 3882c8eac7db1..f715c0408ef80 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -1548,7 +1548,7 @@ Miscellaneous - Bump ``dompurify`` from 2.2.9 to 2.5.6 in /airflow/www (#42263) (#42270) - Correct docstring format in _get_template_context (#42244) (#42272) - Backport: Bump Flask-AppBuilder to ``4.5.2`` (#43309) (#43318) -- Check python version that was used to install pre-commit venvs (#43282) (#43310) +- Check python version that was used to install prek venvs (#43282) (#43310) - Resolve warning in Dataset Alias migration (#43425) Doc Only Changes diff --git a/airflow-core/docs/database-erd-ref.rst b/airflow-core/docs/database-erd-ref.rst index 80673a336ab1b..3f9ea2c77fc65 100644 --- a/airflow-core/docs/database-erd-ref.rst +++ b/airflow-core/docs/database-erd-ref.rst @@ -33,6 +33,6 @@ Here is the current Database schema diagram. `db command `_ for the commands that you can use to manage the migrations. -.. This image is automatically generated by pre-commit via ``scripts/ci/pre_commit/update_er_diagram.py`` +.. This image is automatically generated by prek hook via ``scripts/ci/prek/update_er_diagram.py`` .. image:: img/airflow_erd.svg diff --git a/airflow-core/docs/howto/custom-operator.rst b/airflow-core/docs/howto/custom-operator.rst index d6206166e1211..ad09760c25444 100644 --- a/airflow-core/docs/howto/custom-operator.rst +++ b/airflow-core/docs/howto/custom-operator.rst @@ -349,7 +349,7 @@ Thus, the following example is valid: class MyHelloOperator(HelloOperator): template_fields = "foo" -The limitations above are enforced by a pre-commit named 'validate-operators-init'. +The limitations above are enforced by a prek hook named 'validate-operators-init'. Add template fields with subclassing ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/airflow-core/docs/howto/usage-cli.rst b/airflow-core/docs/howto/usage-cli.rst index a7dcc71abba4d..ce9869b0d419b 100644 --- a/airflow-core/docs/howto/usage-cli.rst +++ b/airflow-core/docs/howto/usage-cli.rst @@ -384,7 +384,7 @@ JSON example output: Testing for DAG Import Errors ----------------------------- The CLI can be used to check whether any discovered dags have import errors via the ``list-import-errors`` subcommand. It is possible to create an automation step which fails if any dags cannot be imported by checking the command output, particularly when used with ``--output`` to generate a standard file format. -For example, the default output when there are no errors is ``No data found``, and the json output is ``[]``. The check can then be run in CI or pre-commit to speed up the review process and testing. +For example, the default output when there are no errors is ``No data found``, and the json output is ``[]``. The check can then be run in CI or as prek hook to speed up the review process and testing. Example command that fails if there are any errors, using `jq `__ to parse the output: diff --git a/airflow-core/docs/installation/supported-versions.rst b/airflow-core/docs/installation/supported-versions.rst index 9b04114e0646f..e06a0302b0b07 100644 --- a/airflow-core/docs/installation/supported-versions.rst +++ b/airflow-core/docs/installation/supported-versions.rst @@ -23,7 +23,7 @@ Version Life Cycle Apache Airflow® version life cycle: - .. This table is automatically updated by pre-commit scripts/ci/pre_commit/supported_versions.py + .. This table is automatically updated by prek hook scripts/ci/prek/supported_versions.py .. Beginning of auto-generated table ========= ===================== ========= =============== ===================== ================ diff --git a/airflow-core/docs/migrations-ref.rst b/airflow-core/docs/migrations-ref.rst index af3ec9a4a90b2..7059d344185f8 100644 --- a/airflow-core/docs/migrations-ref.rst +++ b/airflow-core/docs/migrations-ref.rst @@ -32,7 +32,7 @@ Here's the list of all the Database Migrations that are executed via when you ru - .. This table is automatically updated by pre-commit by ``scripts/ci/pre_commit/migration_reference.py`` + .. This table is automatically updated by prek hook by ``scripts/ci/prek/migration_reference.py`` .. All table elements are scraped from migration files .. Beginning of auto-generated table diff --git a/airflow-core/hatch_build.py b/airflow-core/hatch_build.py index 438767bcd9d5d..87b3df5f0d054 100644 --- a/airflow-core/hatch_build.py +++ b/airflow-core/hatch_build.py @@ -65,7 +65,7 @@ def build_standard(self, directory: str, artifacts: Any, **build_data: Any) -> s self.write_git_version() # run this in the parent directory of the airflow-core (i.e. airflow repo root) work_dir = Path(self.root).parent.resolve() - cmd = ["pre-commit", "run", "--hook-stage", "manual", "compile-ui-assets", "--all-files"] + cmd = ["prek", "run", "--hook-stage", "manual", "compile-ui-assets", "--all-files"] log.warning("Running command: %s", " ".join(cmd)) run(cmd, cwd=work_dir.as_posix(), check=True) dist_path = Path(self.root) / "src" / "airflow" / "ui" / "dist" diff --git a/airflow-core/src/airflow/api_fastapi/gunicorn_config.py b/airflow-core/src/airflow/api_fastapi/gunicorn_config.py index f2c17eef76d4f..c179595fed16a 100644 --- a/airflow-core/src/airflow/api_fastapi/gunicorn_config.py +++ b/airflow-core/src/airflow/api_fastapi/gunicorn_config.py @@ -18,10 +18,20 @@ # under the License. from __future__ import annotations +import sys +from pathlib import Path + import setproctitle from airflow import settings +sys.path.insert(0, str(Path(__file__).parent.resolve())) +from common_prek_utils import ( + initialize_breeze_prek, +) + +initialize_breeze_prek(__name__, __file__) + def post_worker_init(_): """ diff --git a/contributing-docs/02_how_to_communicate.rst b/contributing-docs/02_how_to_communicate.rst index e62341e65ec65..1d0739c2c4095 100644 --- a/contributing-docs/02_how_to_communicate.rst +++ b/contributing-docs/02_how_to_communicate.rst @@ -121,7 +121,7 @@ and loosely organized team. We have both - contributors that commit one commit o more commits. It happens that some people assume informal "stewardship" over parts of code for some time - but at any time we should make sure that the code can be taken over by others, without excessive communication. Setting high requirements for the code (fairly strict code review, static code checks, requirements of -automated tests, pre-commit checks) is the best way to achieve that - by only accepting good quality +automated tests, prek hooks) is the best way to achieve that - by only accepting good quality code. Thanks to full test coverage we can make sure that we will be able to work with the code in the future. So do not be surprised if you are asked to add more tests or make the code cleaner - this is for the sake of maintainability. diff --git a/contributing-docs/03_contributors_quick_start.rst b/contributing-docs/03_contributors_quick_start.rst index 6748d59770ce3..acc33accc3346 100644 --- a/contributing-docs/03_contributors_quick_start.rst +++ b/contributing-docs/03_contributors_quick_start.rst @@ -234,8 +234,8 @@ Forking and cloning Project git config core.autocrlf true -Configuring Pre-commit ----------------------- +Configuring prek +---------------- Before committing changes to github or raising a pull request, the code needs to be checked for certain quality standards such as spell check, code syntax, code formatting, compatibility with Apache License requirements etc. This set of @@ -249,7 +249,7 @@ tests are applied when you commit your code. -To avoid burden on our CI infrastructure and to save time, Pre-commit hooks can be run locally before committing changes. +To avoid burden on our CI infrastructure and to save time, prek hooks can be run locally before committing changes. .. note:: We have recently started to recommend ``uv`` for our local development. @@ -258,7 +258,7 @@ To avoid burden on our CI infrastructure and to save time, Pre-commit hooks can Remember to have global python set to Python >= 3.9 - Python 3.8 is end-of-life already and we've started to use Python 3.9+ features in Airflow and accompanying scripts. -Installing pre-commit is best done with ``uv`` (recommended) or ``pipx``. +Installing prek is best done with ``uv`` (recommended) or ``pipx``. 1. Installing required packages @@ -274,7 +274,7 @@ on macOS, install via brew install libxml2 -2. Installing pre-commit: +2. Installing prek: .. note:: You might need to pass ``--python `` to force the python version if not it uses the latest system python version. @@ -282,15 +282,14 @@ on macOS, install via .. code-block:: bash - uv tool install pre-commit --with pre-commit-uv + uv tool install prek -You can add ``uv`` support for ``pre-commit`` even if you've installed it with ``pipx`` using the commands -(then pre-commit will use ``uv`` to create virtualenvs for the hooks): +or with pipx: .. code-block:: bash - pipx install pre-commit - pipx install inject pre-commit pre-commit-uv # optional, configures pre-commit to use uv to install virtualenvs + pipx install prek + 3. Go to your project directory @@ -299,11 +298,11 @@ You can add ``uv`` support for ``pre-commit`` even if you've installed it with ` cd ~/Projects/airflow -4. Running pre-commit hooks +4. Running prek hooks .. code-block:: bash - pre-commit run --all-files + prek --all-files No-tabs checker......................................................Passed Add license for all SQL files........................................Passed Add license for all other files......................................Passed @@ -327,39 +326,39 @@ You can add ``uv`` support for ``pre-commit`` even if you've installed it with ` Fix End of Files.....................................................Passed ........................................................................... -5. Running pre-commit for selected files +5. Running prek for selected files .. code-block:: bash - pre-commit run --files airflow/utils/decorators.py tests/utils/test_task_group.py + prek --files airflow-core/src/utils/decorators.py tests/utils/test_task_group.py 6. Running specific hook for selected files .. code-block:: bash - pre-commit run black --files airflow/decorators.py tests/utils/test_task_group.py + prek black --files airflow-core/src/airflow/utils/decorators.py airflow-core/tests/unit/utils/test_task_group.py black...............................................................Passed - pre-commit run ruff --files airflow/decorators.py tests/utils/test_task_group.py + prek ruff --files airflow-core/src/airflow/utils/decorators.py airflow-core/tests/unit/utils/test_task_group.py Run ruff............................................................Passed -7. Enabling Pre-commit check before push +7. Enabling prek hook check before push -It will run pre-commit automatically before committing and stops the commit on failure +It will run prek hooks automatically before committing and stops the commit on failure .. code-block:: bash cd ~/Projects/airflow - pre-commit install + prek install git commit -m "Added xyz" -8. To disable Pre-commit +8. To disable prek hooks .. code-block:: bash cd ~/Projects/airflow - pre-commit uninstall + prek uninstall - For more information on this visit |08_static_code_checks.rst| @@ -370,12 +369,12 @@ It will run pre-commit automatically before committing and stops the commit on f - Following are some of the important links of 08_static_code_checks.rst - - |Pre-commit Hooks| + - |Prek Hooks| - .. |Pre-commit Hooks| raw:: html + .. |Prek Hooks| raw:: html - - Pre-commit Hooks + + Prek Hooks - |Running Static Code Checks via Breeze| diff --git a/contributing-docs/05_pull_requests.rst b/contributing-docs/05_pull_requests.rst index a5afb4693806b..fbd74ffb09a20 100644 --- a/contributing-docs/05_pull_requests.rst +++ b/contributing-docs/05_pull_requests.rst @@ -36,13 +36,13 @@ these guidelines: run the tests and `codecov `__ to track coverage. You can set up both for free on your fork. It will help you make sure you do not break the build with your PR and that you help increase coverage. - Also we advise to install locally `pre-commit hooks <08_static_code_checks.rst#pre-commit-hooks>`__ to + Also we advise to install locally `prek hooks <08_static_code_checks.rst#prek-hooks>`__ to apply various checks, code generation and formatting at the time you make a local commit - which gives you near-immediate feedback on things you need to fix before you push your code to the PR, or in many case it will even fix it for you locally so that you can add and commit it straight away. - Follow our project's `Coding style and best practices`_. Usually we attempt to enforce the practices by - having appropriate pre-commits. There are checks amongst them that aren't currently enforced + having appropriate prek hooks. There are checks amongst them that aren't currently enforced programmatically (either because they are too hard or just not yet done). - Maintainers will not merge a PR that regresses linting or does not pass CI tests (unless you have good @@ -62,16 +62,16 @@ these guidelines: maintenance burden during rebase. - Add an `Apache License `__ header to all new files. If you - have ``pre-commit`` installed, pre-commit will do it automatically for you. If you hesitate to install - pre-commit for your local repository - for example because it takes a few seconds to commit your changes, - this one thing might be a good reason to convince anyone to install pre-commit. + have ``prek`` installed, prej will do it automatically for you. If you hesitate to install + prek for your local repository - for example because it takes a few seconds to commit your changes, + this one thing might be a good reason to convince anyone to install prek. - If your PR adds functionality, make sure to update the docs as part of the same PR, not only code and tests. Docstring is often sufficient. Make sure to follow the Sphinx compatible standards. - Make sure your code fulfills all the `static code checks <08_static_code_checks.rst#static-code-checks>`__ we have in our code. The easiest way - to make sure of that is - again - to install `pre-commit hooks <08_static_code_checks.rst#pre-commit-hooks>`__ + to make sure of that is - again - to install `prek hooks <08_static_code_checks.rst#prek-hooks>`__ - Make sure your PR is small and focused on one change only - avoid adding unrelated changes, mixing adding features and refactoring. Keeping to that rule will make it easier to review your PR and will make @@ -271,7 +271,7 @@ In such cases we can usually do something like this self.my_field = my_field The reason for doing it is that we are working on a cleaning up our code to have -`pre-commit hook <../scripts/ci/pre_commit/validate_operators_init.py>`__ +`prek hook <../scripts/ci/prek/validate_operators_init.py>`__ that will make sure all the cases where logic (such as validation and complex conversion) is not done in the constructor are detected in PRs. diff --git a/contributing-docs/06_development_environments.rst b/contributing-docs/06_development_environments.rst index 5ce138a3f1c83..67c7734a3d85d 100644 --- a/contributing-docs/06_development_environments.rst +++ b/contributing-docs/06_development_environments.rst @@ -57,9 +57,9 @@ Limitations: Breeze container-based solution provides a reproducible environment that is consistent with other developers. -- You are **STRONGLY** encouraged to also install and use `pre-commit hooks <08_static_code_checks.rst#pre-commit-hooks>`_ +- You are **STRONGLY** encouraged to also install and use `prek hooks <08_static_code_checks.rst#prek-hooks>`_ for your local virtualenv development environment. - Pre-commit hooks can speed up your development cycle a lot. + Prek hooks can speed up your development cycle a lot. Typically you can connect your local virtualenv environments easily with your IDE and use it for development: diff --git a/contributing-docs/07_local_virtualenv.rst b/contributing-docs/07_local_virtualenv.rst index d565309f2cf28..491c4cff915cb 100644 --- a/contributing-docs/07_local_virtualenv.rst +++ b/contributing-docs/07_local_virtualenv.rst @@ -70,7 +70,7 @@ The ``uv`` utility is a build frontend tool that is designed to manage python, v and testing of Python projects. It is a modern tool that is designed to work with PEP 517/518 compliant projects and it is much faster than "reference" ``pip`` tool. It has extensive support to not only create development environment but also to manage python versions, development environments, workspaces and Python tools used -to develop Airflow (via ``uv tool`` command - such as ``pre-commit`` and others, you can also use ``uv tool`` +to develop Airflow (via ``uv tool`` command - such as ``prek`` and others, you can also use ``uv tool`` to install ``breeze`` - containerized development environment for Airflow that we use to reproduce the CI environment locally and to run release-management and certain development tasks. @@ -247,7 +247,7 @@ dependencies, and sometimes ``development dependencies`` (in ``dev`` dependency to run tests and are installed automatically when you install environment with ``uv-sync``. If you want to add another dependency to a provider, you should add it to corresponding ``pyproject.toml``, -add the files to your commit with ``git add`` and run ``pre-commit run`` to update generated dependencies. +add the files to your commit with ``git add`` and run ``prek`` to update generated dependencies. Note that in the future we will remove that step. For ``uv`` it's simple, you need to run ``uv sync`` in providers directory after you modified diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 9d7fed230c7c1..fe254d838054e 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -19,439 +19,195 @@ Static code checks ================== The static code checks in Airflow are used to verify that the code meets certain quality standards. -All the static code checks can be run through pre-commit hooks. +All the static code checks can be run through prek hooks. -The pre-commit hooks perform all the necessary installation when you run them -for the first time. See the table below to identify which pre-commit checks require the Breeze Docker images. +The prek hooks perform all the necessary installation when you run them +for the first time. See the table below to identify which prek checks require the Breeze Docker images. You can also run the checks via `Breeze <../dev/breeze/doc/README.rst>`_ environment. **The outline for this document in GitHub is available at top-right corner button (with 3-dots and 3 lines).** -Pre-commit hooks ----------------- +Prek hooks +---------- Pre-commit hooks help speed up your local development cycle and place less burden on the CI infrastructure. -Consider installing the pre-commit hooks as a necessary prerequisite. +Consider installing the prek hooks as a necessary prerequisite. -The pre-commit hooks by default only check the files you are currently working on and make -them fast. Yet, these checks use exactly the same environment as the CI tests +The hooks by default only check the files you are currently working on (and are staged) which makes the +checks rather fast. Yet, these checks use exactly the same environment as the CI tests use. So, you can be sure your modifications will also work for CI if they pass -pre-commit hooks. +prek hooks. -We have integrated the fantastic `pre-commit `__ framework -in our development workflow. To install and use it, you need at least Python 3.9 locally. +We have integrated the `prek `__ framework +in our development workflow. It can be installed in various ways and does not even need ``pip`` or -Installing pre-commit hooks ---------------------------- +``python`` to be installed. It is a drop-in replacement for the legacy ``pre-commit`` tool, but it is +much faster and more feature-rich. It is written in Rust and it is designed to install environments in parallel, +so it is much faster than the ``pre-commit`` tool. + +Installing prek hooks +--------------------- + +It is the best to use prek hooks when you have your local virtualenv for +Airflow activated since then prek hooks and other dependencies are +automatically installed. You can also install the prek hooks manually using ``uv`` or ``pipx``. -It is the best to use pre-commit hooks when you have your local virtualenv for -Airflow activated since then pre-commit hooks and other dependencies are -automatically installed. You can also install the pre-commit hooks manually using ``uv`` or ``pip``. +.. code-block:: bash + + uv tool install prek .. code-block:: bash - uv tool install pre-commit + pipx install prek + +Since we have a lot of hooks and sometimes you want to run them individually, it's advised to install +auto-completion for the ``prek`` command. You can do it by adding the following line to your +``.bashrc`` or ``.zshrc`` file: + +For bash: .. code-block:: bash - pip install pre-commit + eval "$(COMPLETE=bash prek)" # for bash + +For zsh: + +.. code-block:: zsh -After installation, pre-commit hooks are run automatically when you commit the code and they will + eval "$(COMPLETE=zsh prek)" + +Similarly for other shells like fish, powershell, etc. + +After installation, prek hooks are run automatically when you commit the code and they will only run on the files that you change during your commit, so they are usually pretty fast and do -not slow down your iteration speed on your changes. There are also ways to disable the ``pre-commits`` +not slow down your iteration speed on your changes. There are also ways to disable the prek hooks temporarily when you commit your code with ``--no-verify`` switch or skip certain checks that you find -to much disturbing your local workflow. See `Available pre-commit checks <#available-pre-commit-checks>`_ -and `Using pre-commit <#using-pre-commit>`_ +to much disturbing your local workflow. See `Using prek <#using-prek>`_ -The pre-commit hooks use several external linters that need to be installed before pre-commit is run. +The ``prek`` hooks use several external linters that need to be installed before prek is run. Each of the checks installs its own environment, so you do not need to install those, but there are some checks that require locally installed binaries. On Linux, you typically install them with ``sudo apt install``, on macOS - with ``brew install``. -The current list of prerequisites is limited to ``xmllint``: +The current list of prerequisites is limited to ``xmllint`` and ``golang`` if you want to modify +the Golang code.: -- on Linux, install via ``sudo apt install libxml2-utils`` -- on macOS, install via ``brew install libxml2`` +- on Linux, install via ``sudo apt install libxml2-utils golang`` +- on macOS, install via ``brew install libxml2 golang`` -Some pre-commit hooks also require the Docker Engine to be configured as the static -checks are executed in the Docker environment (See table in the -`Available pre-commit checks <#available-pre-commit-checks>`_ . You should build the images -locally before installing pre-commit checks as described in `Breeze docs <../dev/breeze/doc/README.rst>`__. +Some prek hooks also require the Docker Engine to be configured as the static +checks are executed in the Docker environment. You should build the images +locally before installing prek checks as described in `Breeze docs <../dev/breeze/doc/README.rst>`__. Sometimes your image is outdated and needs to be rebuilt because some dependencies have been changed. -In such cases, the Docker-based pre-commit will inform you that you should rebuild the image. +In such cases, the Docker-based prek will inform you that you should rebuild the image. -In case you do not have your local images built, the pre-commit hooks fail and provide -instructions on what needs to be done. +Enabling prek hooks +------------------- -Enabling pre-commit hooks -------------------------- +To turn on prek checks for ``commit`` operations in git, enter: -To turn on pre-commit checks for ``commit`` operations in git, enter: +.. code-block:: bash + + prek install + +To install the checks also for ``pre-push`` operations, enter: .. code-block:: bash - pre-commit install + prek install -t pre-push +For details on advanced usage of the install method, use: -To install the checks also for ``pre-push`` operations, enter: +.. code-block:: bash + + prek install --help + +.. note:: + + The ``prek`` tool is a drop-in replacement for the legacy ``pre-commit`` tool - much faster and more + feature-rich, If you have already installed ``pre-commit`` to handle your hooks, you can run + ``prek install -f`` to replace the existing ``pre-commit`` hooks with the ``prek`` hooks. + +Available prek hooks +-------------------- + +You can see the list of available hooks by running: .. code-block:: bash - pre-commit install -t pre-push + prek list +You can also see more details about the hooks by running: -For details on advanced usage of the install method, use: +.. code-block:: bash + + prek list --verbose + +And if you want to see the details of a particular hook, you can run: .. code-block:: bash - pre-commit install --help + prek list --verbose -Available pre-commit checks ---------------------------- +When you install auto-completion, you can also use the tab-completion to see the available hooks. + +Using prek +---------- -This table lists pre-commit hooks used by Airflow. The ``Image`` column indicates which hooks -require Breeze Docker image to be built locally. - - .. BEGIN AUTO-GENERATED STATIC CHECK LIST - -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ID | Description | Image | -+===========================================================+========================================================+=========+ -| bandit | bandit | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| blacken-docs | Run black on docs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-aiobotocore-optional | Check if aiobotocore is an optional dependency only | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-airflow-k8s-not-used | Check airflow.kubernetes imports are not used | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-airflow-providers-bug-report-template | Sort airflow-bug-report provider list | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-apache-license-rat | Check if licenses are OK for Apache | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-base-operator-partial-arguments | Check BaseOperator and partial() arguments | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-base-operator-usage | * Check BaseOperator core imports | | -| | * Check BaseOperatorLink core imports | | -| | * Check BaseOperator other imports | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-boring-cyborg-configuration | Checks for Boring Cyborg configuration consistency | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-breeze-top-dependencies-limited | Check top-level breeze deps | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-builtin-literals | Require literal syntax when initializing builtins | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-changelog-format | Check changelog format | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-changelog-has-no-duplicates | Check changelogs for duplicate entries | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-cncf-k8s-only-for-executors | Check cncf.kubernetes imports used for executors only | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-code-deprecations | Check deprecations categories in decorators | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-common-compat-used-for-openlineage | Check common.compat is used for OL deprecated classes | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-core-deprecation-classes | Verify usage of Airflow deprecation classes in core | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-daysago-import-from-utils | days_ago imported from airflow.utils.dates | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-decorated-operator-implements-custom-name | Check @task decorator implements custom_operator_name | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-default-configuration | Check the default configuration | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-deferrable-default | Check and fix default value of default_deferrable | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-docstring-param-types | Check that docstrings do not specify param types | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-example-dags-urls | Check that example dags url include provider versions | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-executables-have-shebangs | Check that executables have shebang | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-extra-packages-references | Checks setup extra packages | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-extras-order | Check order of extras in Dockerfile | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-fab-migrations | Check no migration is done on FAB related table | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-for-inclusive-language | Check for language that we do not accept as community | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-get-lineage-collector-providers | Check providers import hook lineage code from compat | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-hooks-apply | Check if all hooks apply to the repository | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-imports-in-providers | Check imports in providers | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-incorrect-use-of-LoggingMixin | Make sure LoggingMixin is not used alone | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-init-decorator-arguments | Sync model __init__ and decorator arguments | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-integrations-list-consistent | Sync integrations list with docs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-lazy-logging | Check that all logging methods are lazy | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-links-to-example-dags-do-not-use-hardcoded-versions | Verify no hard-coded version in example dags | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-merge-conflict | Check that merge conflicts are not being committed | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-min-python-version | Check minimum Python version | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-newsfragments-are-valid | Check newsfragments are valid | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-no-airflow-deprecation-in-providers | Do not use DeprecationWarning in providers | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-no-providers-in-core-examples | No providers imports in core example DAGs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-only-new-session-with-provide-session | Check NEW_SESSION is only used with @provide_session | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-persist-credentials-disabled-in-github-workflows | Check persistent creds in workflow files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-pre-commit-information-consistent | Validate hook IDs & names and sync with docs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-provide-create-sessions-imports | Check session util imports | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-provider-docs-valid | Validate provider doc files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-provider-yaml-valid | Validate provider.yaml files | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-providers-subpackages-init-file-exist | Provider subpackage init files are there | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-pydevd-left-in-code | Check for pydevd debug statements accidentally left | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-revision-heads-map | Check that the REVISION_HEADS_MAP is up-to-date | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-safe-filter-usage-in-html | Don't use safe in templates | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-significant-newsfragments-are-valid | Check significant newsfragments are valid | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-sql-dependency-common-data-structure | Check dependency of SQL providers | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-start-date-not-used-in-defaults | start_date not in default_args | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-system-tests-present | Check if system tests have required segments of code | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-system-tests-tocs | Check that system tests is properly added | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-taskinstance-tis-attrs | Check that TI and TIS have the same attributes | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-template-context-variable-in-sync | Sync template context variable refs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-template-fields-valid | Check templated fields mapped in operators/sensors | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-tests-in-the-right-folders | Check if tests are in the right folders | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-tests-unittest-testcase | Unit tests do not inherit from unittest.TestCase | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-urlparse-usage-in-code | Don't use urlparse in code | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-xml | Check XML files with xmllint | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| check-zip-file-is-not-committed | Check no zip files are committed | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| codespell | Run codespell | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| compile-fab-assets | Compile FAB provider assets | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| compile-ui-assets | Compile ui assets (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| compile-ui-assets-dev | Compile ui assets in dev mode (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| create-missing-init-py-files-tests | Create missing init.py files in tests | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| debug-statements | Detect accidentally committed debug statements | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| detect-private-key | Detect if private key is added to the repository | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| doctoc | Add TOC for Markdown and RST files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| end-of-file-fixer | Make sure that there is an empty line at the end | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| flynt | Run flynt string format converter for Python | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-airflow-diagrams | Generate airflow diagrams | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-openapi-spec | Generate the FastAPI API spec | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-openapi-spec-fab | Generate the FastAPI API spec for FAB | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-pypi-readme | Generate PyPI README | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-tasksdk-datamodels | Generate Datamodels for TaskSDK client | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| generate-volumes-for-sources | Generate volumes for docker compose | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| identity | Print checked files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| insert-license | * Add license for all SQL files | | -| | * Add license for all RST files | | -| | * Add license for CSS/JS/JSX/PUML/TS/TSX | | -| | * Add license for all Shell files | | -| | * Add license for all toml files | | -| | * Add license for all Python files | | -| | * Add license for all XML files | | -| | * Add license for all Helm template files | | -| | * Add license for all YAML files except Helm templates | | -| | * Add license for all Markdown files | | -| | * Add license for all other files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| kubeconform | Kubeconform check on our helm chart | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| lint-chart-schema | Lint chart/values.schema.json file | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| lint-dockerfile | Lint Dockerfile | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| lint-helm-chart | Lint Helm Chart | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| lint-json-schema | * Lint JSON Schema files | | -| | * Lint NodePort Service | | -| | * Lint Docker compose files | | -| | * Lint chart/values.schema.json | | -| | * Lint chart/values.yaml | | -| | * Lint config_templates/config.yml | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| lint-markdown | Run markdownlint | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mixed-line-ending | Detect if mixed line ending is used (\r vs. \r\n) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-airflow-core | * Run mypy for airflow-core | * | -| | * Run mypy for airflow-core (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-airflow-ctl | * Run mypy for airflow-ctl | * | -| | * Run mypy for airflow-ctl (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-dev | * Run mypy for dev | * | -| | * Run mypy for dev (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-devel-common | * Run mypy for devel-common | * | -| | * Run mypy for devel-common (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-providers | * Run mypy for providers | * | -| | * Run mypy for providers (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| mypy-task-sdk | * Run mypy for task-sdk | * | -| | * Run mypy for task-sdk (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| pretty-format-json | Format JSON files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| pylint | pylint | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| python-no-log-warn | Check if there are no deprecate log warn | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| replace-bad-characters | Replace bad characters | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| rst-backticks | Check if RST files use double backticks for code | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ruff | Run 'ruff' for extremely fast Python linting | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ruff-format | Run 'ruff format' | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| shellcheck | Check Shell scripts syntax correctness | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| trailing-whitespace | Remove trailing whitespace at end of line | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ts-compile-lint-simple-auth-manager-ui | Compile / format / lint simple auth manager UI | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| ts-compile-lint-ui | Compile / format / lint UI | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-black-version | Update black versions everywhere (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-breeze-cmd-output | Update breeze docs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-breeze-readme-config-hash | Update Breeze README.md with config files hash | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-chart-dependencies | Update chart dependencies to latest (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-er-diagram | Update ER diagram | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-in-the-wild-to-be-sorted | Sort INTHEWILD.md alphabetically | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-inlined-dockerfile-scripts | Inline Dockerfile and Dockerfile.ci scripts | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-installed-providers-to-be-sorted | Sort and uniquify installed_providers.txt | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-installers-and-pre-commit | Update installers and pre-commit to latest (manual) | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-local-yml-file | Update mounts in the local yml file | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-migration-references | Update migration ref doc | * | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-providers-build-files | Update providers build files | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-providers-dependencies | Update dependencies for providers | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-pyproject-toml | Update Airflow's meta-package pyproject.toml | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-reproducible-source-date-epoch | Update Source Date Epoch for reproducible builds | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-spelling-wordlist-to-be-sorted | Sort spelling_wordlist.txt | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-supported-versions | Updates supported versions in documentation | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-vendored-in-k8s-json-schema | Vendor k8s definitions into values.schema.json | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-version | Update versions in docs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| validate-operators-init | No templated field logic checks in operator __init__ | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| yamllint | Check YAML files with yamllint | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ -| zizmor | Run zizmor to check for github workflow syntax errors | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ - - .. END AUTO-GENERATED STATIC CHECK LIST - -Using pre-commit ----------------- - -After installation, pre-commit hooks are run automatically when you commit the -code. But you can run pre-commit hooks manually as needed. +After installation, prek hooks are run automatically when you commit the +code. But you can run prek hooks manually as needed. - Run all checks on your staged files by using: .. code-block:: bash - pre-commit run + prek -- Run only mypy check on your staged files (in ``airflow/`` excluding providers) by using: +- Run only mypy check on your staged airflow and dev files by specifying the + ``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be specified): .. code-block:: bash - pre-commit run mypy-airflow + prek mypy-airflow-core mypy-dev -- Run only mypy checks on all files by using: +- Run only mypy airflow checks on all "airflow-core" files by using: .. code-block:: bash - pre-commit run mypy-airflow --all-files - + prek mypy-airflow-core --all-files - Run all checks on all files by using: .. code-block:: bash - pre-commit run --all-files - + prek --all-files - Run all checks only on files modified in the last locally available commit in your checked out branch: .. code-block:: bash - pre-commit run --source=HEAD^ --origin=HEAD + prek --last-commit + +- Run all checks only on files modified in your last branch that is targeted to be merged into the main branch: +.. code-block:: bash + + prek --from-ref main -- Show files modified automatically by pre-commit when pre-commits automatically fix errors +- Show files modified automatically by prek when prek automatically fix errors .. code-block:: bash - pre-commit run --show-diff-on-failure + prek --show-diff-on-failure - Skip one or more of the checks by specifying a comma-separated list of checks to skip in the SKIP variable: .. code-block:: bash - SKIP=mypy-airflow-core,ruff pre-commit run --all-files + SKIP=mypy-airflow-core,ruff prek --all-files You can always skip running the tests by providing ``--no-verify`` flag to the @@ -462,30 +218,48 @@ To check other usage types of the pre-commit framework, see `Pre-commit website Disabling particular checks --------------------------- -In case you have a problem with running particular ``pre-commit`` check you can still continue using the -benefits of having ``pre-commit`` installed, with some of the checks disabled. In order to disable +In case you have a problem with running particular ``prek`` check you can still continue using the +benefits of having ``prek`` installed, with some of the checks disabled. In order to disable checks you might need to set ``SKIP`` environment variable to coma-separated list of checks to skip. For example, when you want to skip some checks (ruff/mypy for example), you should be able to do it by setting ``export SKIP=ruff,mypy-airflow-core,``. You can also add this to your ``.bashrc`` or ``.zshrc`` if you do not want to set it manually every time you enter the terminal. In case you do not have breeze image configured locally, you can also disable all checks that require breeze -the image by setting ``SKIP_BREEZE_PRE_COMMITS`` to "true". This will mark the tests as "green" automatically +the image by setting ``SKIP_BREEZE_PREK_HOOKS`` to "true". This will mark the tests as "green" automatically when run locally (note that those checks will anyway run in CI). -Manual pre-commits ------------------- +Manual prek hooks +----------------- Most of the checks we run are configured to run automatically when you commit the code. However, there are some checks that are not run automatically and you need to run them manually. Those checks are marked with ``manual`` in the ``Description`` column in the table below. You can run -them manually by running ``pre-commit run --hook-stage manual ``. +them manually by running ``prek --hook-stage manual ``. + +Special pin-versions prek +------------------------- + +There is a separate prek ``pin-versions`` prek hook which is used to pin versions of +GitHub Actions in the CI workflows. + +This action requires ``GITHUB_TOKEN`` to be set, otherwise you might hit the rate limits with GitHub API, it +is also configured in a separate ``.prek-config.yaml`` file in the +``.github`` directory as it requires Python 3.11 to run. It is not run automatically +when you commit the code but in runs as a separate job in the CI. However, you can run it +manually by running: + +.. code-block:: bash + + export GITHUB_TOKEN=YOUR_GITHUB_TOKEN + prek -c .github/.pre-commit-config.yaml --all-files --hook-stage manual --verbose + Mypy checks ----------- When we run mypy checks locally when committing a change, one of the ``mypy-*`` checks is run, ``mypy-airflow``, -``mypy-dev``, ``mypy-providers``, ``mypy-airflow-ctl``, depending on the files you are changing. The mypy checks +``mypy-dev``, ``mypy-providers`` depending on the files you are changing. The mypy checks are run by passing those changed files to mypy. This is way faster than running checks for all files (even if mypy cache is used - especially when you change a file in airflow core that is imported and used by many files). However, in some cases, it produces different results than when running checks for the whole set @@ -498,19 +272,19 @@ command (example for ``airflow`` files): .. code-block:: bash - pre-commit run --hook-stage manual mypy- --all-files + prek --hook-stage manual mypy- --all-files For example: .. code-block:: bash - pre-commit run --hook-stage manual mypy-airflow --all-files + prek --hook-stage manual mypy-airflow --all-files To show unused mypy ignores for any providers/airflow etc, eg: run below command: .. code-block:: bash export SHOW_UNUSED_MYPY_WARNINGS=true - pre-commit run --hook-stage manual mypy-airflow --all-files + prek --hook-stage manual mypy-airflow --all-files MyPy uses a separate docker-volume (called ``mypy-cache-volume``) that keeps the cache of last MyPy execution in order to speed MyPy checks up (sometimes by order of magnitude). While in most cases MyPy @@ -518,99 +292,6 @@ will handle refreshing the cache when and if needed, there are some cases when i is the hard problem in computer science). This might happen for example when we upgrade MyPY. In such cases you might need to manually remove the cache volume by running ``breeze down --cleanup-mypy-cache``. -Running static code checks via Breeze -------------------------------------- - -The static code checks can be launched using the Breeze environment. - -You run the static code checks via ``breeze static-check`` or commands. - -You can see the list of available static checks either via ``--help`` flag or by using the autocomplete -option. - -Run the ``mypy`` check for the currently staged changes (in ``airflow/`` excluding providers): - -.. code-block:: bash - - breeze static-checks --type mypy-airflow - -Run the ``mypy`` check for all files: - -.. code-block:: bash - - breeze static-checks --type mypy-airflow --all-files - -Run the ``ruff`` check for the ``tests/core.py`` file with verbose output: - -.. code-block:: bash - - breeze static-checks --type ruff --file tests/core.py --verbose - -Run the ``ruff`` check for the ``tests.core`` package with verbose output: - -.. code-block:: bash - - breeze static-checks --type ruff --file tests/core/* --verbose - -Run the ``ruff-format`` check for the files ``airflow/example_dags/example_bash_operator.py`` and -``airflow/example_dags/example_python_operator.py``: - -.. code-block:: bash - - breeze static-checks --type ruff-format --file airflow/example_dags/example_bash_operator.py \ - airflow/example_dags/example_python_operator.py - -Run all checks for the currently staged files: - -.. code-block:: bash - - breeze static-checks - -Run all checks for all files: - -.. code-block:: bash - - breeze static-checks --all-files - -Run all checks for last commit: - -.. code-block:: bash - - breeze static-checks --last-commit - -Run all checks for all changes in my branch since branched from main: - -.. code-block:: bash - - breeze static-checks --type mypy-airflow --only-my-changes - -More examples can be found in -`Breeze documentation <../dev/breeze/doc/03_developer_tasks.rst#running-static-checks>`_ - - -Debugging pre-commit check scripts requiring image --------------------------------------------------- - -Those commits that use Breeze docker image might sometimes fail, depending on your operating system and -docker setup, so sometimes it might be required to run debugging with the commands. This is done via -two environment variables ``VERBOSE`` and ``DRY_RUN``. Setting them to "true" will respectively show the -commands to run before running them or skip running the commands. - -Note that you need to run pre-commit with --verbose command to get the output regardless of the status -of the static check (normally it will only show output on failure). - -Printing the commands while executing: - -.. code-block:: bash - - VERBOSE="true" pre-commit run --verbose ruff - -Just performing dry run: - -.. code-block:: bash - - DRY_RUN="true" pre-commit run --verbose ruff - ----------- Once your code passes all the static code checks, you should take a look at `Testing documentation <09_testing.rst>`__ diff --git a/contributing-docs/12_provider_distributions.rst b/contributing-docs/12_provider_distributions.rst index f85f1daa4ba7f..434ebccbe99a2 100644 --- a/contributing-docs/12_provider_distributions.rst +++ b/contributing-docs/12_provider_distributions.rst @@ -107,9 +107,9 @@ needs to be updated after you change dependencies: .. code:: bash - breeze static-checks --type update-providers-dependencies --all-files + prek update-providers-dependencies --all-files -If you have ``pre-commit`` installed, this will be done automatically for you when you commit the changes and +If you have ``prek`` installed, this will be done automatically for you when you commit the changes and you should do it before you make a PR with such changed dependency changes Also, you should rebuild the image ``breeze ci-image build`` or answer ``y`` when you are asked to rebuild the @@ -121,7 +121,7 @@ Provider's cross-dependencies Some of the providers have cross-dependencies with other providers distributions. This typically happens for transfer operators where operators use hooks from the other providers in case they are transferring data between the providers. The list of dependencies is maintained -(automatically with the ``update-providers-dependencies`` pre-commit) in the +(automatically with the ``update-providers-dependencies`` prek hook) in the ``generated/provider_dependencies.json``. Cross-dependencies between providers are converted into optional dependencies (extras) - if @@ -165,7 +165,7 @@ in the previous chapter. However when they are locally developed, together with of discovery of the providers is based on ``provider.yaml`` file that is placed in the top-folder of the provider. The ``provider.yaml`` is the single source of truth for the provider metadata and it is there where you should add and remove dependencies for providers (following by running -``update-providers-dependencies`` pre-commit to synchronize the dependencies with ``pyproject.toml`` +``update-providers-dependencies`` prek hook to synchronize the dependencies with ``pyproject.toml`` of Airflow). The ``provider.yaml`` file is compliant with the schema that is available in @@ -278,7 +278,7 @@ and documented. Part of the documentation is ``provider.yaml`` file ``integratio ``version`` information. This information is stripped-out from provider info available at runtime, however it is used to automatically generate documentation for the provider. -If you have pre-commits installed, pre-commit will warn you and let you know what changes need to be +If you have prek installed, it will warn you and let you know what changes need to be done in the ``provider.yaml`` file when you add a new Operator, Hooks, Sensor or Transfer. You can also take a look at the other ``provider.yaml`` files as examples. diff --git a/contributing-docs/14_metadata_database_updates.rst b/contributing-docs/14_metadata_database_updates.rst index d0d903604990a..49de75f812862 100644 --- a/contributing-docs/14_metadata_database_updates.rst +++ b/contributing-docs/14_metadata_database_updates.rst @@ -36,12 +36,12 @@ database schema that you have made. To generate a new migration file, run the fo Generating ~/airflow/airflow/migrations/versions/a1e23c41f123_add_new_field_to_db.py -Note that migration file names are standardized by pre-commit hook ``update-migration-references``, so that they sort alphabetically and indicate -the Airflow version in which they first appear (the alembic revision ID is removed). As a result you should expect to see a pre-commit failure +Note that migration file names are standardized by prek hook ``update-migration-references``, so that they sort alphabetically and indicate +the Airflow version in which they first appear (the alembic revision ID is removed). As a result you should expect to see a prek failure on the first attempt. Just stage the modified file and commit again (or run the hook manually before committing). -After your new migration file is run through pre-commit it will look like this: +After your new migration file is run through prek hook it will look like this: .. code-block:: @@ -70,8 +70,8 @@ To resolve these conflicts: .. code-block:: bash - pre-commit run update-migration-references --all - pre-commit run update-er-diagram --all + prek update-migration-references --all-files + prek update-er-diagram --all-files 3. Add the updated files to the staging area and continue with the rebase. diff --git a/contributing-docs/16_adding_api_endpoints.rst b/contributing-docs/16_adding_api_endpoints.rst index 1881e7a2ba8a2..54cbe6163e722 100644 --- a/contributing-docs/16_adding_api_endpoints.rst +++ b/contributing-docs/16_adding_api_endpoints.rst @@ -18,7 +18,7 @@ Adding a New API Endpoint in Apache Airflow =========================================== -This documentation outlines the steps required to add a new API endpoint in Apache Airflow. It includes implementing the logic, running pre-commit checks, and documenting the changes. +This documentation outlines the steps required to add a new API endpoint in Apache Airflow. It includes implementing the logic, running prek hooks, and documenting the changes. **The outline for this document in GitHub is available at top-right corner button (with 3-dots and 3 lines).** @@ -67,19 +67,19 @@ Step 2: Add tests for your Endpoints Step 3: Documentation --------------------- -Documentation is built automatically by FastAPI and our pre-commit hooks. Verify by going to ``/docs`` that the documentation is clear and appears as expected (body and return types, query params, validation) +Documentation is built automatically by FastAPI and our prek hooks. Verify by going to ``/docs`` that the documentation is clear and appears as expected (body and return types, query params, validation) -Step 4: Run Pre-commit Hooks ------------------------------ -1. Ensure all code meets the project's quality standards by running pre-commit hooks. -2. Pre-commit hooks include static code checks, formatting, and other validations. +Step 4: Run prek Hooks +---------------------- +1. Ensure all code meets the project's quality standards by running prek hooks. +2. Prek hooks include static code checks, formatting, and other validations. 3. Persisted openapi spec is located in ``v2-rest-api-generated.yaml` and a hook will take care of updating it based on your new endpoint, you just need to add and commit the change. -4. Run the following command to execute all pre-commit hooks: +4. Run the following command to execute all prek hooks: .. code-block:: bash - pre-commit run --all-files + prek --all-files Optional: Adding Pydantic Model @@ -99,7 +99,7 @@ In some cases, you may need to define additional models for new data structures. These models are defined to structure and validate the data handled by the API. Once defined, these models will automatically be added to the OpenAPI spec file as long as they are actually used by one endpoint. -After adding or modifying Pydantic models, make sure to run the pre-commit hooks again to update any generated files. +After adding or modifying Pydantic models, make sure to run the prek hooks again to update any generated files. Situational: Legacy Endpoint Migration -------------------------------------- diff --git a/contributing-docs/17_architecture_diagrams.rst b/contributing-docs/17_architecture_diagrams.rst index 73d01440642d9..f82b2a2e0c9f6 100644 --- a/contributing-docs/17_architecture_diagrams.rst +++ b/contributing-docs/17_architecture_diagrams.rst @@ -20,7 +20,7 @@ Architecture Diagrams We started to use (and gradually convert old diagrams to use it) `Diagrams `_ as our tool of choice to generate diagrams. The diagrams are generated from Python code and can be -automatically updated when the code changes. The diagrams are generated using pre-commit hooks (See +automatically updated when the code changes. The diagrams are generated using prek hooks (See static checks below) but they can also be generated manually by running the corresponding Python code. To run the code you need to install the dependencies in the virtualenv you use to run it: @@ -28,10 +28,10 @@ To run the code you need to install the dependencies in the virtualenv you use t system (``brew install graphviz`` on macOS for example). The source code of the diagrams are next to the generated diagram, the difference is that the source -code has ``.py`` extension and the generated diagram has ``.png`` extension. The pre-commit hook ``generate-airflow-diagrams`` +code has ``.py`` extension and the generated diagram has ``.png`` extension. The prek hook ``generate-airflow-diagrams`` will look for ``diagram_*.py`` files in the ``docs`` subdirectories to find them and runs them when the sources changed and the diagrams are not up to date (the -pre-commit will automatically generate an .md5sum hash of the sources and store it next to the diagram +prek hook will automatically generate an .md5sum hash of the sources and store it next to the diagram file). In order to generate the diagram manually you can run the following command: @@ -44,19 +44,19 @@ You can also generate all diagrams by: .. code-block:: bash - pre-commit run generate-airflow-diagrams + prek generate-airflow-diagrams or with Breeze: .. code-block:: bash - breeze static-checks --type generate-airflow-diagrams --all-files + prek generate-airflow-diagrams --all-files When you iterate over a diagram, you can also setup a "save" action in your IDE to run the python file automatically when you save the diagram file. Once you've done iteration and you are happy with the diagram, you can commit the diagram, the source -code and the .md5sum file. The pre-commit hook will then not run the diagram generation until the +code and the .md5sum file. The prek hook will then not run the diagram generation until the source code for it changes. ------ diff --git a/contributing-docs/18_contribution_workflow.rst b/contributing-docs/18_contribution_workflow.rst index ccc6ec5a42827..5e9f1f1a0ed8b 100644 --- a/contributing-docs/18_contribution_workflow.rst +++ b/contributing-docs/18_contribution_workflow.rst @@ -40,7 +40,7 @@ In general, your contribution includes the following stages: 2. Create a `local virtualenv <07_local_virtualenv.rst>`_, initialize the `Breeze environment <../dev/breeze/doc/README.rst>`__, and - install `pre-commit framework <08_static_code_checks.rst#pre-commit-hooks>`__. + install `prek tool <08_static_code_checks.rst#prek-hooks>`__. If you want to add more changes in the future, set up your fork and enable GitHub Actions. 3. Join `devlist `__ @@ -176,9 +176,9 @@ Step 4: Prepare PR * Modify the class and add necessary code and unit tests. * Run and fix all the `static checks <08_static_code_checks.rst>`__. If you have - `pre-commits installed <08_static_code_checks.rst#pre-commit-hooks>`__, + `prek installed <08_static_code_checks.rst#prek-hooks>`__, this step is automatically run while you are committing your code. If not, you can do it manually - via ``git add`` and then ``pre-commit run``. + via ``git add`` and then ``prek``. * Run the appropriate tests as described in `Testing documentation <09_testing.rst>`__. diff --git a/contributing-docs/testing/integration_tests.rst b/contributing-docs/testing/integration_tests.rst index 401b383ab3bc9..a5d6c6deb873e 100644 --- a/contributing-docs/testing/integration_tests.rst +++ b/contributing-docs/testing/integration_tests.rst @@ -298,9 +298,6 @@ Then, create the integration test file under ``tests/integration`` - remember to and to use the ``@pytest.mark.integration`` decorator. It is recommended to define setup and teardown methods (``setup_method`` and ``teardown_method``, respectively) - you could look at existing integration tests to learn more. -Before pushing to GitHub, make sure to run static checks (``breeze static-checks --only-my-changes``) to apply linters -on the Python logic, as well as to update the commands images under ``dev/breeze/docs/images``. - When writing integration tests for components that also require Kerberos, you could enforce auto-enabling the latter by updating ``compose_file()`` method in ``airflow_breeze.params.shell_params.ShellParams``. For example, to ensure that Kerberos is active for ``trino`` integration tests, the following code has been introduced: diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index b1f935847f824..2cf85f6454bb8 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -272,7 +272,7 @@ pipx install -e ./dev/breeze - Set your version in `airflow/__init__.py` (without the RC tag). - Run `git commit` without a message to update versions in `docs`. -- Add supported Airflow version to `./scripts/ci/pre_commit/supported_versions.py` and let pre-commit do the job again. +- Add supported Airflow version to `./scripts/ci/prek/supported_versions.py` and let prek do the job again. - Replace the versions in `README.md` about installation and verify that installation instructions work fine. - Add entry for default python version to `BASE_PROVIDERS_COMPATIBILITY_CHECKS` in `src/airflow_breeze/global_constants.py` with the new Airflow version, and empty exclusion for providers. This list should be updated later when providers @@ -1101,7 +1101,7 @@ EOF This includes: -- Modify `./scripts/ci/pre_commit/supported_versions.py` and let pre-commit do the job. +- Modify `./scripts/ci/prek/supported_versions.py` and let prek do the job. - For major/minor release, update version in `airflow/__init__.py` and `docs/docker-stack/` to the next likely minor version release. - Sync `RELEASE_NOTES.rst` (including deleting relevant `newsfragments`) and `README.md` changes. - Updating `Dockerfile` with the new version. diff --git a/dev/README_RELEASE_HELM_CHART.md b/dev/README_RELEASE_HELM_CHART.md index 94234db7a2c8a..3acbc46ed0e7c 100644 --- a/dev/README_RELEASE_HELM_CHART.md +++ b/dev/README_RELEASE_HELM_CHART.md @@ -153,7 +153,7 @@ annotations: ``` Make sure that all the release notes changes are submitted as PR and merged. Changes in release notes should -also automatically (via `pre-commit` trigger updating of the [reproducible_build.yaml](../chart/reproducible_build.yaml)) +also automatically (via `prek` trigger updating of the [reproducible_build.yaml](../chart/reproducible_build.yaml)) file which is uses to reproducibly build the chart package and source tarball. You can leave the k8s environment now: diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 95c6ed78be7cc..3e4defc837d99 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -131,10 +131,10 @@ Note that when you update dependencies/lock them you should commit the changes i See [uv documentation](https://docs.astral.sh/uv/getting-started/) for more details on using `uv`. -PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. +PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PREK. --------------------------------------------------------------------------------------------------------- -Package config hash: 45d0d735430fd1c151071f8ce20ac90b65d5bdc4317678ce1a03004a9f0853745a7b8cbf70520fc55afac15f742e961eeb7f5ffbf948f1b230f5cba788fceb65 +Package config hash: 9d908346f49e07bb1c179b2fc1f172f52d8b553cd8dbc380ce8ee1d735130cee203813e588ace9814d3f70ce8e7ba05b4b904bb7bbddda331338e940ed93c94a --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/doc/01_installation.rst b/dev/breeze/doc/01_installation.rst index 9b54ed17d7542..3f39fd643b92c 100644 --- a/dev/breeze/doc/01_installation.rst +++ b/dev/breeze/doc/01_installation.rst @@ -125,7 +125,7 @@ Docker in WSL 2 If you're experiencing errors such as ``ERROR: for docker-compose_airflow_run Cannot create container for service airflow: not a directory`` when starting Breeze after the first time or an error like ``docker: Error response from daemon: not a directory. - See 'docker run --help'.`` when running the pre-commit tests, you may need to consider + See 'docker run --help'.`` when running the prek tests, you may need to consider `installing Docker directly in WSL 2 `_ instead of using Docker Desktop for Windows. diff --git a/dev/breeze/doc/03_developer_tasks.rst b/dev/breeze/doc/03_developer_tasks.rst index 5ea9a48884ae7..c0a55f22de5f2 100644 --- a/dev/breeze/doc/03_developer_tasks.rst +++ b/dev/breeze/doc/03_developer_tasks.rst @@ -265,32 +265,30 @@ as the short hand operator. Running static checks --------------------- -You can run static checks via Breeze. You can also run them via pre-commit command but with auto-completion -Breeze makes it easier to run selective static checks. If you press after the static-check and if -you have auto-complete setup you should see auto-completable list of all checks available. +You can run static checks via prek. For example, this following command: .. code-block:: bash - breeze static-checks --type mypy-airflow + prek mypy-airflow will run mypy check for currently staged files inside ``airflow/`` excluding providers. Selecting files to run static checks on --------------------------------------- -Pre-commits run by default on staged changes that you have locally changed. It will run it on all the +Prek hooks run by default on staged changes that you have locally changed. It will run it on all the files you run ``git add`` on and it will ignore any changes that you have modified but not staged. If you want to run it on all your modified files you should add them with ``git add`` command. With ``--all-files`` you can run static checks on all files in the repository. This is useful when you want to be sure they will not fail in CI, or when you just rebased your changes and want to -re-run latest pre-commits on your changes, but it can take a long time (few minutes) to wait for the result. +re-run latest prek hooks on your changes, but it can take a long time (few minutes) to wait for the result. .. code-block:: bash - breeze static-checks --type mypy-airflow --all-files + prek mypy-airflow --all-files The above will run mypy check for all files. @@ -299,58 +297,35 @@ specifying (can be multiple times) ``--file`` flag. .. code-block:: bash - breeze static-checks --type mypy-airflow --file airflow/utils/code_utils.py --file airflow/utils/timeout.py + prek mypy-airflow --file airflow/utils/code_utils.py --file airflow/utils/timeout.py The above will run mypy check for those to files (note: autocomplete should work for the file selection). However, often you do not remember files you modified and you want to run checks for files that belong -to specific commits you already have in your branch. You can use ``breeze static check`` to run the checks +to specific commits you already have in your branch. You can use ``prek`` to run the checks only on changed files you have already committed to your branch - either for specific commit, for last commit, for all changes in your branch since you branched off from main or for specific range of commits you choose. .. code-block:: bash - breeze static-checks --type mypy-airflow --last-commit + prek mypy-airflow --last-commit The above will run mypy check for all files in the last commit in your branch. .. code-block:: bash - breeze static-checks --type mypy-airflow --only-my-changes - -The above will run mypy check for all commits in your branch which were added since you branched off from main. - -.. code-block:: bash - - breeze static-checks --type mypy-airflow --commit-ref 639483d998ecac64d0fef7c5aa4634414065f690 - -The above will run mypy check for all files in the 639483d998ecac64d0fef7c5aa4634414065f690 commit. -Any ``commit-ish`` reference from Git will work here (branch, tag, short/long hash etc.) - -.. code-block:: bash - - breeze static-checks --type identity --verbose --from-ref HEAD^^^^ --to-ref HEAD + prek identity --verbose --from-ref HEAD^^^^ --to-ref HEAD The above will run the check for the last 4 commits in your branch. You can use any ``commit-ish`` references in ``--from-ref`` and ``--to-ref`` flags. - -These are all available flags of ``static-checks`` command: - -.. image:: ./images/output_static-checks.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_static-checks.svg - :width: 100% - :alt: Breeze static checks - - .. note:: When you run static checks, some of the artifacts (mypy_cache) is stored in docker-compose volume so that it can speed up static checks execution significantly. However, sometimes, the cache might get broken, in which case you should run ``breeze down`` to clean up the cache. - .. note:: You cannot change Python version for static checks that are run within Breeze containers. @@ -440,7 +415,7 @@ command takes care about it. This is needed when you want to run API server insi Note -This command requires the ``pre-commit`` tool, which should be installed by following `this guide <../../../contributing-docs/03_contributors_quick_start.rst#configuring-pre-commit>`__. +This command requires the ``prek`` tool, which should be installed by following `this guide <../../../contributing-docs/03_contributors_quick_start.rst#configuring-prek>`__. Breeze cleanup -------------- diff --git a/dev/breeze/doc/04_troubleshooting.rst b/dev/breeze/doc/04_troubleshooting.rst index f9828ef1b700e..64f5dbdd52ad6 100644 --- a/dev/breeze/doc/04_troubleshooting.rst +++ b/dev/breeze/doc/04_troubleshooting.rst @@ -99,30 +99,28 @@ or ERROR: Package 'blacken-docs' requires a different Python: 3.8.18 not in '>=3.9' -It means that your pre-commit hook is installed with (already End-Of-Life) Python 3.8 and you should reinstall -it and clean pre-commit cache. +It means that your prek hook is installed with (already End-Of-Life) Python 3.8 and you should reinstall +it and clean prek cache. -This can be done with ``uv tool`` to install ``pre-commit``) +This can be done with ``uv tool`` to install ``prek``) .. code-block:: bash - uv tool uninstall pre-commit - uv tool install pre-commit --python 3.9 --force --with pre-commit-uv - pre-commit clean - pre-commit install + uv tool uninstall prek + uv tool install prek --python 3.9 --force + prek clean + prek install You can also use ``pipx`` .. code-block:: bash - pipx uninstall pre-commit - pipx install pre-commit --python $(which python3.9) --force - # This one allows pre-commit to use uv for venvs installed by pre-commit - pipx inject pre-commit pre-commit-uv # optionally if you want to use uv to install virtualenvs - pre-commit clean - pre-commit install + pipx uninstall prek + pipx install prek --python $(which python3.9) --force + prek clean + prek install -If you installed ``pre-commit`` differently, you should remove and reinstall +If you installed ``prek`` differently, you should remove and reinstall it (and clean cache) following the way you installed it. @@ -132,7 +130,7 @@ Bad Interpreter Error with ``pipx`` If you are experiencing bad interpreter errors ``zsh: /Users/eladkal/.local/bin/breeze: bad interpreter: /Users/eladkal/.local/pipx/venvs/apache-airflow-breeze/bin/python: no such file or directory`` -try to run ``pipx list`` to view which packages has bad interpreter (it can be more than just breeze, for example pre-commit) +try to run ``pipx list`` to view which packages has bad interpreter (it can be more than just breeze, for example prek) you can fix these errors by running ``pipx reinstall-all`` ETIMEDOUT Error @@ -170,7 +168,7 @@ When running ``breeze start-airflow``, either normally or in ``dev-mode``, the f The asset compilation failed. Exiting. - [INFO] Locking pre-commit directory + [INFO] Locking prek directory Error 1 returned diff --git a/dev/breeze/doc/05_test_commands.rst b/dev/breeze/doc/05_test_commands.rst index a51013e171a25..0ca1c072b4a20 100644 --- a/dev/breeze/doc/05_test_commands.rst +++ b/dev/breeze/doc/05_test_commands.rst @@ -317,6 +317,25 @@ through ``breeze testing docker-compose-tests`` command. The docker-compose tests are in ``docker-tests/`` folder in the main repo. +Running task-sdk integration tests +.................................. + +You can use Breeze to run the task sdk integration tests. Those tests are run using Production image by default +and the tests are running with the docker-compose we have for task-sdk tests. + +.. image:: ./images/output_testing_docker-compose-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_task-sdk-integration-tests.svg + :width: 100% + :alt: Breeze testing task-sdk-integration-tests + +You can also iterate over those tests with pytest command but unlike regular unit tests, they need to be run in +a local venv. You can build the prod image with breeze and that will be used by default if present to run the tests. + +You can override the ``DOCKER_IMAGE`` environment variable to point to the image to test using the +``breeze testing task-sdk-integration-tests`` command. + +The task-sdk tests are in ``task-sdk-tests/`` folder in the main repo. + Running Kubernetes tests ------------------------ diff --git a/dev/breeze/doc/07_breeze_maintenance_tasks.rst b/dev/breeze/doc/07_breeze_maintenance_tasks.rst index f139040375ed0..83f502106d60a 100644 --- a/dev/breeze/doc/07_breeze_maintenance_tasks.rst +++ b/dev/breeze/doc/07_breeze_maintenance_tasks.rst @@ -29,7 +29,7 @@ Regenerating documentation SVG screenshots This documentation contains exported SVG screenshots with "help" of their commands and parameters. You can regenerate those images that need to be regenerated because their commands changed (usually after the breeze code has been changed) via ``regenerate-command-images`` command. Usually this is done -automatically via pre-commit, but sometimes (for example when ``rich`` or ``rich-click`` library changes) +automatically via ``prek``, but sometimes (for example when ``rich`` or ``rich-click`` library changes) you need to regenerate those images. You can add ``--force`` flag (or ``FORCE="true"`` environment variable to regenerate all images (not diff --git a/dev/breeze/doc/10_advanced_breeze_topics.rst b/dev/breeze/doc/10_advanced_breeze_topics.rst index 3cd22855db2fe..364d0a1d7a1df 100644 --- a/dev/breeze/doc/10_advanced_breeze_topics.rst +++ b/dev/breeze/doc/10_advanced_breeze_topics.rst @@ -149,16 +149,13 @@ If you want to add core dependency that should always be installed - you need to to ``dependencies`` section. If you want to add it to one of the optional core extras, you should add it in the extra definition in ``pyproject.toml`` (you need to find out where it is defined). If you want to add it to one of the providers, you need to add it to the ``provider.yaml`` file in the provider -directory - but remember that this should be followed by running pre-commit that will automatically update +directory - but remember that this should be followed by running prek that will automatically update the ``pyproject.toml`` with the new dependencies as the ``provider.yaml`` files are not used directly, they are used to update ``pyproject.toml`` file: .. code-block:: bash - pre-commit run update-providers-dependencies --all-files - -You can also run the pre-commit by ``breeze static-checks --type update-providers-dependencies --all-files`` -command - which provides autocomplete. + prek update-providers-dependencies --all-files After you've updated the dependencies, you need to rebuild the image: @@ -191,15 +188,11 @@ scripts are present in ``scripts/docker`` directory and are aptly (!) named ``in of the apt dependencies are installed in the ``install_os_dependencies.sh``, but some are installed in other scripts (for example ``install_postgres.sh`` or ``install_mysql.sh``). -After you modify the dependencies in the scripts, you need to inline them by running pre-commit: +After you modify the dependencies in the scripts, you need to inline them by running prek: .. code-block:: bash - pre-commit run update-inlined-dockerfile-scripts --all-files - -You can also run the pre-commit by ``breeze static-checks --type update-inlined-dockerfile-scripts --all-files`` -command - which provides autocomplete. - + prek update-inlined-dockerfile-scripts --all-files After you've updated the dependencies, you need to rebuild the image: diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index f17ae7e3f8a90..f107794290685 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -444,8 +444,7 @@ can be used for CI images: | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | | `AIRFLOW_PIP_VERSION` | `25.2` | `pip` version used. | | `AIRFLOW_UV_VERSION` | `0.8.8` | `uv` version used. | -| `AIRFLOW_PRE_COMMIT_VERSION` | `4.3.0` | `pre-commit` version used. | -| `AIRFLOW_PRE_COMMIT_UV_VERSION` | `4.1.4` | `pre-commit-uv` version used. | +| `AIRFLOW_PREK_VERSION` | `0.0.26` | `prek` version used. | | `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | | `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 7f0b64dcbc726..056f3a10c5eab 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -24,7 +24,7 @@ - [Selective CI Checks](#selective-ci-checks) - [Groups of files that selective check make decisions on](#groups-of-files-that-selective-check-make-decisions-on) - [Selective check decision rules](#selective-check-decision-rules) - - [Skipping pre-commits (Static checks)](#skipping-pre-commits-static-checks) + - [Skipping prek hooks (Static checks)](#skipping-prek-hooks-static-checks) - [Suspended providers](#suspended-providers) - [Selective check outputs](#selective-check-outputs) - [Committer vs. Non-committer PRs](#committer-vs-non-committer-prs) @@ -111,7 +111,7 @@ together using `pytest-xdist` (pytest-xdist distributes the tests among parallel * If there are no files left in sources after matching the test types and Kubernetes files, then apparently some Core/Other files have been changed. This automatically adds all test types to execute. This is done because changes in core might impact all the other test types. -* if `CI Image building` is disabled, only basic pre-commits are enabled - no 'image-depending` pre-commits +* if `CI Image building` is disabled, only basic prek hooks are enabled - no 'image-depending` prek hooks are enabled. * If there are some build dependencies changed (`hatch_build.py` and updated system dependencies in the `pyproject.toml` - then `upgrade to newer dependencies` is enabled. @@ -121,12 +121,12 @@ together using `pytest-xdist` (pytest-xdist distributes the tests among parallel changed, also providers docs are built because all providers depend on airflow docs. If any of the docs build python files changed or when build is "canary" type in main - all docs packages are built. -## Skipping pre-commits (Static checks) +## Skipping prek hooks (Static checks) -Our CI always run pre-commit checks with `--all-files` flag. This is in order to avoid cases where -different check results are run when only subset of files is used. This has an effect that the pre-commit +Our CI always run prek checks with `--all-files` flag. This is in order to avoid cases where +different check results are run when only subset of files is used. This has an effect that the prek tests take a long time to run when all of them are run. Selective checks allow to save a lot of time -for those tests in regular PRs of contributors by smart detection of which pre-commits should be skipped +for those tests in regular PRs of contributors by smart detection of which prek hooks should be skipped when some files are not changed. Those are the rules implemented: * The `identity` check is always skipped (saves space to display all changed files in CI) @@ -137,7 +137,7 @@ when some files are not changed. Those are the rules implemented: * check-provider-yaml-valid * lint-helm-chart * mypy-providers -* If "full tests" mode is detected, no more pre-commits are skipped - we run all of them +* If "full tests" mode is detected, no more prek hooks are skipped - we run all of them * The following checks are skipped if those files are not changed: * if no `All Providers Python files` changed - `mypy-providers` check is skipped * if no `All Airflow Python files` changed - `mypy-airflow` check is skipped @@ -170,7 +170,7 @@ GitHub Actions to pass the list of parameters to a command to execute | all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.9 3.10 | * | | all-versions | If set to true, then all python, k8s, DB versions are used for tests. | false | | | basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | -| ci-image-build | Whether CI image build is needed | true | | +| ci-image-build | Whether CI image build is needed | true | | | core-test-types-list-as-strings-in-json | Which test types should be run for unit tests for core | API Always Providers | * | | debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | | default-branch | Which branch is default for the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | @@ -230,7 +230,7 @@ GitHub Actions to pass the list of parameters to a command to execute | amd-runners | List of labels assigned for runners for that build for public AMD runners | \["ubuntu-22.04"\] | | | arm-runners | List of labels assigned for runners for that build for public ARM runners | \["ubuntu-22.04-arm"\] | | | selected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * | -| skip-pre-commits | Which pre-commits should be skipped during the static-checks run | flynt,identity | | +| skip-prek-hooks | Which prek hooks should be skipped during the static-checks run | flynt,identity | | | skip-providers-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | | sqlite-exclude | Which versions of Sqlite to exclude for tests as JSON array | [] | | | testable-core-integrations | List of core integrations that are testable in the build as JSON array | \['celery', 'kerberos'\] | | @@ -244,7 +244,7 @@ When we decided whether to run `full tests` we do not check (directly) if provid even if they are single source of truth for provider dependencies and when you add a dependency there, the environment changes and generally full tests are advised. -This is because provider.yaml change will automatically trigger (via `update-provider-dependencies` pre-commit) +This is because provider.yaml change will automatically trigger (via `update-provider-dependencies` prek) generation of `generated/provider_dependencies.json` and `pyproject.toml` gets updated as well. This is a far better indication if we need to run full tests than just checking if provider.yaml files changed, because provider.yaml files contain more information than just dependencies - they are the single source of truth diff --git a/dev/breeze/doc/ci/05_workflows.md b/dev/breeze/doc/ci/05_workflows.md index 8db97a96a8dea..ed7211a3e6125 100644 --- a/dev/breeze/doc/ci/05_workflows.md +++ b/dev/breeze/doc/ci/05_workflows.md @@ -202,7 +202,7 @@ Windows. they are needed (basically, when Python code has been modified). `(6)` On non-main builds some of the static checks that are related to -Providers are skipped via selective checks (`skip-pre-commits` check). +Providers are skipped via selective checks (`skip-prek-hooks` check). `(7)` On non-main builds the unit tests, docs and integration tests for providers are skipped via selective checks. diff --git a/dev/breeze/doc/ci/07_running_ci_locally.md b/dev/breeze/doc/ci/07_running_ci_locally.md index 6724008b8cc99..a0422f928b4b5 100644 --- a/dev/breeze/doc/ci/07_running_ci_locally.md +++ b/dev/breeze/doc/ci/07_running_ci_locally.md @@ -36,7 +36,7 @@ The main goal of the CI philosophy we have that no matter how complex the test and integration infrastructure, as a developer you should be able to reproduce and re-run any of the failed checks locally. One part -of it are pre-commit checks, that allow you to run the same static +of it are prek checks, that allow you to run the same static checks in CI and locally, but another part is the CI environment which is replicated locally with Breeze. @@ -163,8 +163,8 @@ Those variables are used to control the initialization of the environment in the | Variable | Option | Local dev | CI | Comment | |---------------------------------|------------------------------------|-----------|-----------|-----------------------------------------------------------------------------| | MOUNT_SOURCES | --mount-sources | | skip | Whether to mount the local sources into the container. | -| SKIP_ENVIRONMENT_INITIALIZATION | --skip-environment-initialization | false (*) | false (*) | Skip initialization of test environment (*) set to true in pre-commits. | -| SKIP_IMAGE_UPGRADE_CHECK | --skip-image-upgrade-check | false (*) | false (*) | Skip checking if image should be upgraded (*) set to true in pre-commits. | +| SKIP_ENVIRONMENT_INITIALIZATION | --skip-environment-initialization | false (*) | false (*) | Skip initialization of test environment (*) set to true in prek hooks. | +| SKIP_IMAGE_UPGRADE_CHECK | --skip-image-upgrade-check | false (*) | false (*) | Skip checking if image should be upgraded (*) set to true in prek hooks. | | SKIP_PROVIDERS_TESTS | | false | false | Skip running provider integration tests (in non-main branch). | | SKIP_SSH_SETUP | | false | false (*) | Skip setting up SSH server for tests. (*) set to true in GitHub CodeSpaces. | | VERBOSE_COMMANDS | | false | false | Whether every command executed in docker should be printed. | diff --git a/dev/breeze/doc/images/output-commands-hash.txt b/dev/breeze/doc/images/output-commands-hash.txt index 407f5d55ae7f0..c27896f7b526b 100644 --- a/dev/breeze/doc/images/output-commands-hash.txt +++ b/dev/breeze/doc/images/output-commands-hash.txt @@ -1,4 +1,4 @@ -# This file is automatically generated by pre-commit. If you have a conflict with this file +# This file is automatically generated by prek. If you have a conflict with this file # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. main:ffb1a766b791beaf5f8a983587db870f diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index 5171dd9625ca3..8aa291bc175d0 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -1,4 +1,4 @@ - +