From c8411fd8c959382cab5eb7a14fe61b36cbce0625 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 27 Mar 2024 02:31:22 +0100 Subject: [PATCH] Review and set variables in our workflows where appropriate Some of the variable names (importnat only for fork builds) have not been moved to the new workflows. This PR updates variabels used in various workflows to be set consistently. --- .../workflows/additional-ci-image-checks.yml | 6 +++ .../workflows/additional-prod-image-tests.yml | 6 +++ .github/workflows/build-images.yml | 10 ++--- .github/workflows/check-providers.yml | 24 +++++++++--- .github/workflows/ci-image-build.yml | 11 ++++++ .github/workflows/ci.yml | 15 +------- .github/workflows/docs.yml | 10 +++-- .github/workflows/finalize-tests.yml | 4 ++ .github/workflows/helm-tests.yml | 3 ++ .github/workflows/integration-tests.yml | 3 ++ .github/workflows/k8s-tests.yml | 3 ++ .github/workflows/prod-image-build.yml | 10 ++++- .github/workflows/push-image-cache.yml | 38 +++++++++++++------ .github/workflows/run-unit-tests.yml | 6 ++- ...checks-mypy-and-constraints-generation.yml | 9 +++-- 15 files changed, 115 insertions(+), 43 deletions(-) diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index f73fe08ec7ec3..c2ee18b647857 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -123,7 +123,10 @@ jobs: UPGRADE_TO_NEWER_DEPENDENCIES: false PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }} PYTHON_VERSION: ${{ inputs.default-python-version }} + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" if: inputs.canary-run == 'true' && inputs.branch == 'main' steps: - name: "Cleanup repo" @@ -173,7 +176,10 @@ jobs: env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" IMAGE_TAG: ${{ inputs.image-tag }} + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" # for now release check is failing and we need to fix it but we don't want to block the PRs if: false steps: diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 8eb8699e6c2e6..06d2da0b71a70 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -91,7 +91,10 @@ jobs: name: "Test examples of POD image building" runs-on: ["ubuntu-22.04"] env: + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash @@ -134,7 +137,10 @@ jobs: env: IMAGE_TAG: "${{ inputs.image-tag }}" PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index a1d5c0937c67a..19c3b28b1ed96 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -28,20 +28,20 @@ permissions: packages: read env: ANSWER: "yes" - DB_RESET: "true" - VERBOSE: "true" - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_USERNAME: ${{ github.actor }} # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the # Airflow one is going to be used CONSTRAINTS_GITHUB_REPOSITORY: >- ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} # This token is WRITE one - pull_request_target type of events always have the WRITE token + DB_RESET: "true" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" - USE_SUDO: "true" INCLUDE_SUCCESS_OUTPUTS: "true" + USE_SUDO: "true" + VERBOSE: "true" concurrency: group: build-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml index e307a59657ca3..b5e47eb9b5730 100644 --- a/.github/workflows/check-providers.yml +++ b/.github/workflows/check-providers.yml @@ -61,9 +61,13 @@ jobs: name: "Provider packages wheel build and verify" runs-on: ${{fromJSON(inputs.runs-on)}} env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - IMAGE_TAG: "${{ inputs.image-tag }}" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + IMAGE_TAG: "${{ inputs.image-tag }}" + INCLUDE_NOT_READY_PROVIDERS: "true" + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash @@ -125,9 +129,13 @@ jobs: name: "Provider packages sdist build and install" runs-on: ${{fromJSON(inputs.runs-on)}} env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - IMAGE_TAG: "${{ inputs.image-tag }}" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + IMAGE_TAG: "${{ inputs.image-tag }}" + INCLUDE_NOT_READY_PROVIDERS: "true" + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash @@ -193,10 +201,14 @@ jobs: matrix: include: ${{fromJson(inputs.providers-compatibility-checks)}} env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} IMAGE_TAG: "${{ inputs.image-tag }}" - PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" + INCLUDE_NOT_READY_PROVIDERS: "true" + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" VERSION_SUFFIX_FOR_PYPI: "dev0" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERBOSE: "true" if: inputs.skip-provider-tests != 'true' steps: - name: "Cleanup repo" diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index e384d7cf837e6..d28a34daf5ac5 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -115,8 +115,11 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} VERSION_SUFFIX_FOR_PYPI: "dev0" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} USE_UV: ${{ inputs.use-uv }} + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash @@ -198,7 +201,15 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" DOCKER_CACHE: ${{ inputs.docker-cache }} INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} + # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the + # Airflow one is going to be used + CONSTRAINTS_GITHUB_REPOSITORY: >- + ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && + secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} + # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} PUSH: ${{ inputs.push-image }} VERBOSE: "true" if: inputs.do-build == 'true' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 992a49ac43094..57fddf61e48d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,22 +34,11 @@ permissions: # changes. If it is for a fork, then the token is read-only anyway. packages: write env: - DB_RESET: "true" - VERBOSE: "true" GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_USERNAME: ${{ github.actor }} - # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the - # Airflow one is going to be used - CONSTRAINTS_GITHUB_REPOSITORY: >- - ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && - secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} - # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" - USE_SUDO: "true" - INCLUDE_NOT_READY_PROVIDERS: "true" - AIRFLOW_ENABLE_AIP_44: "true" - MOUNT_SOURCES: "skip" + VERBOSE: "true" concurrency: group: ci-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 88b74625fae3c..67261e31b769a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -59,10 +59,14 @@ jobs: matrix: flag: ["--docs-only", "--spellcheck-only"] env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" - IMAGE_TAG: "${{ inputs.image-tag }}" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + IMAGE_TAG: "${{ inputs.image-tag }}" + INCLUDE_NOT_READY_PROVIDERS: "true" + INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 52c451545beee..fd31998e0501d 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -81,6 +81,10 @@ jobs: DEBUG_RESOURCES: ${{ inputs.debug-resources}} PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }} IMAGE_TAG: ${{ inputs.image-tag }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" if: inputs.upgrade-to-newer-dependencies != 'false' steps: - name: "Cleanup repo" diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index a729d391b58c4..50b7bb3ab669d 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -55,7 +55,10 @@ jobs: JOB_ID: "helm-tests" USE_XDIST: "true" IMAGE_TAG: "${{ inputs.image-tag }}" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index dcc0b4fbbb068..991fc7590fc36 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -84,7 +84,10 @@ jobs: SKIP_PROVIDER_TESTS: "${{ inputs.skip-provider-tests }}" ENABLE_COVERAGE: "${{ inputs.run-coverage}}" DEBUG_RESOURCES: "${{ inputs.debug-resources }}" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 1482dd050d1e6..7e09efa994038 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -65,7 +65,10 @@ jobs: DEBUG_RESOURCES: ${{ inputs.debug-resources }} INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }} IMAGE_TAG: ${{ inputs.image-tag }} + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 9556633c40e68..5c5b59c0ea6df 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -125,9 +125,17 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} VERSION_SUFFIX_FOR_PYPI: "dev0" INCLUDE_NOT_READY_PROVIDERS: "true" + # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the + # Airflow one is going to be used + CONSTRAINTS_GITHUB_REPOSITORY: >- + ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && + secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} + # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERBOSE: "true" + GITHUB_USERNAME: ${{ github.actor }} USE_UV: ${{ inputs.use-uv }} + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 186b2f09878d0..13e7d12be3140 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -82,18 +82,26 @@ jobs: matrix: python: ${{ fromJson(inputs.python-versions) }} env: + COMMIT_SHA: ${{ github.sha }} + # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the + # Airflow one is going to be used + CONSTRAINTS_GITHUB_REPOSITORY: >- + ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && + secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} + # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one DEBIAN_VERSION: ${{ inputs.debian-version }} - INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} - USE_UV: ${{ inputs.use-uv }} DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} - UPGRADE_TO_NEWER_DEPENDENCIES: "false" DOCKER_CACHE: ${{ inputs.docker-cache }} - VERSION_SUFFIX_FOR_PYPI: "dev0" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COMMIT_SHA: ${{ github.sha }} - VERBOSE: "true" + GITHUB_USERNAME: ${{ github.actor }} INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" + INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} + USE_UV: ${{ inputs.use-uv }} + UPGRADE_TO_NEWER_DEPENDENCIES: "false" + VERBOSE: "true" + VERSION_SUFFIX_FOR_PYPI: "dev0" steps: - name: "Cleanup repo" shell: bash @@ -132,17 +140,25 @@ jobs: matrix: python: ${{ fromJson(inputs.python-versions) }} env: + COMMIT_SHA: ${{ github.sha }} + # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the + # Airflow one is going to be used + CONSTRAINTS_GITHUB_REPOSITORY: >- + ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && + secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} + # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one DEBIAN_VERSION: ${{ inputs.debian-version }} - INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} - USE_UV: ${{ inputs.use-uv }} DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} - UPGRADE_TO_NEWER_DEPENDENCIES: "false" DOCKER_CACHE: ${{ inputs.docker-cache }} - VERSION_SUFFIX_FOR_PYPI: "dev0" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COMMIT_SHA: ${{ github.sha }} + GITHUB_USERNAME: ${{ github.actor }} + INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} + UPGRADE_TO_NEWER_DEPENDENCIES: "false" + USE_UV: ${{ inputs.use-uv }} VERBOSE: "true" + VERSION_SUFFIX_FOR_PYPI: "dev0" if: inputs.include-prod-images == 'true' steps: - name: "Cleanup repo" diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 88fc8b612eda0..a54019da211ac 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -124,10 +124,14 @@ jobs: AIRFLOW_ENABLE_AIP_44: "${{ inputs.enable-aip-44 }}" BACKEND: "${{ inputs.backend }}" BACKEND_VERSION: "${{ matrix.backend-version }}" + DB_RESET: "true" DEBUG_RESOURCES: "${{ inputs.debug-resources }}" DOWNGRADE_SQLALCHEMY: "${{ inputs.downgrade-sqlalchemy }}" DOWNGRADE_PENDULUM: "${{ inputs.downgrade-pendulum }}" ENABLE_COVERAGE: "${{ inputs.run-coverage }}" + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} IMAGE_TAG: "${{ inputs.image-tag }}" INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }} # yamllint disable rule:line-length @@ -137,7 +141,7 @@ jobs: PYDANTIC: "${{ inputs.pydantic }}" PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" UPGRADE_BOTO: "${{ inputs.upgrade-boto }}" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERBOSE: "true" steps: - name: "Cleanup repo" shell: bash diff --git a/.github/workflows/static-checks-mypy-and-constraints-generation.yml b/.github/workflows/static-checks-mypy-and-constraints-generation.yml index 3130c23a2596a..fbd33f240ac79 100644 --- a/.github/workflows/static-checks-mypy-and-constraints-generation.yml +++ b/.github/workflows/static-checks-mypy-and-constraints-generation.yml @@ -77,11 +77,14 @@ jobs: name: Generate constraints ${{ inputs.python-versions-list-as-string }} runs-on: ['ubuntu-22.04'] env: - PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }} DEBUG_RESOURCES: ${{ inputs.debug-resources }} - VERSION_SUFFIX_FOR_PYPI: "dev0" - INCLUDE_SUCCESS_OUTPUTS: "true" + GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + INCLUDE_SUCCESS_OUTPUTS: "true" + PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }} + VERBOSE: "true" + VERSION_SUFFIX_FOR_PYPI: "dev0" if: inputs.ci-image-build == 'true' steps: - name: "Cleanup repo"