From 880ec2524603e9061b91a2e0045310d8f382c2a7 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 16 Nov 2023 16:29:53 +0100 Subject: [PATCH] Use reproducible builds for provider packages Flit allows to build reproducible packages (packages that can be compared bit-by-bit) providing that source date epoch is set to repeatable value when package is built. This PR implements reproducibility of our builds by freezing the documentation preparation time in provider.yaml as "source date epoch" and always using it when building the package. This way anyone using breeze to build the package will have exactly the same binary package produced, which will make it way easier to verify if the packages are ready for release by the PMC member. We will no longer have to check the sources, PMC members will simply need to build the same packages locally using breeze and see if the generated packages are exactly the same. That also includes permissions bits - flit sets permissions of the generated packages to "no permissions for other/group" in order to get bit-to-bit reproducibility - because on some systems umask is set differently by default and created file would be different because of the permission bit. This caused problems when building docker images - so we had it changed when moving packages to 'dist' folder, but this PR changes it so that CI build does it when moving packages from dist to docker-context-files instead The "source-date-epoch" fields have been regenerated in this PR as well. Also this PR replaces `lru_cache` method of storing output of `get_provider_metadata_packages` with custom-stored dictionary - thanks to that instead of invalidating whole cache of providers metadata refreshed from yaml files we can refresh individual provider metadata entries after they have been updated. This saves a lot of time for validation - because every time when provider yaml is updated we need to re-read it and re-validate it with json schema, with this change we only do it for the updated provider yaml - which saves about 0.5 a second per provider yaml update and when you update all provides it is done way faster. --- .github/actions/build-prod-images/action.yml | 4 +- .github/workflows/ci.yml | 4 +- airflow/provider.yaml.schema.json | 11 +++++ airflow/providers/airbyte/provider.yaml | 1 + airflow/providers/alibaba/provider.yaml | 1 + airflow/providers/amazon/provider.yaml | 1 + airflow/providers/apache/beam/provider.yaml | 1 + .../providers/apache/cassandra/provider.yaml | 1 + airflow/providers/apache/drill/provider.yaml | 1 + airflow/providers/apache/druid/provider.yaml | 1 + airflow/providers/apache/flink/provider.yaml | 1 + airflow/providers/apache/hdfs/provider.yaml | 1 + airflow/providers/apache/hive/provider.yaml | 1 + airflow/providers/apache/impala/provider.yaml | 1 + airflow/providers/apache/kafka/provider.yaml | 1 + airflow/providers/apache/kylin/provider.yaml | 1 + airflow/providers/apache/livy/provider.yaml | 1 + airflow/providers/apache/pig/provider.yaml | 1 + airflow/providers/apache/pinot/provider.yaml | 1 + airflow/providers/apache/spark/provider.yaml | 1 + airflow/providers/apache/sqoop/provider.yaml | 1 + airflow/providers/apprise/provider.yaml | 1 + airflow/providers/arangodb/provider.yaml | 1 + airflow/providers/asana/provider.yaml | 1 + .../providers/atlassian/jira/provider.yaml | 1 + airflow/providers/celery/provider.yaml | 1 + airflow/providers/cloudant/provider.yaml | 1 + .../providers/cncf/kubernetes/provider.yaml | 1 + airflow/providers/cohere/provider.yaml | 1 + airflow/providers/common/io/provider.yaml | 1 + airflow/providers/common/sql/provider.yaml | 1 + airflow/providers/daskexecutor/provider.yaml | 1 + airflow/providers/databricks/provider.yaml | 1 + airflow/providers/datadog/provider.yaml | 1 + airflow/providers/dbt/cloud/provider.yaml | 1 + airflow/providers/dingding/provider.yaml | 1 + airflow/providers/discord/provider.yaml | 1 + airflow/providers/docker/provider.yaml | 1 + airflow/providers/elasticsearch/provider.yaml | 1 + airflow/providers/exasol/provider.yaml | 1 + airflow/providers/facebook/provider.yaml | 1 + airflow/providers/ftp/provider.yaml | 1 + airflow/providers/github/provider.yaml | 1 + airflow/providers/google/provider.yaml | 1 + airflow/providers/grpc/provider.yaml | 1 + airflow/providers/hashicorp/provider.yaml | 1 + airflow/providers/http/provider.yaml | 1 + airflow/providers/imap/provider.yaml | 1 + airflow/providers/influxdb/provider.yaml | 1 + airflow/providers/jdbc/provider.yaml | 1 + airflow/providers/jenkins/provider.yaml | 1 + .../providers/microsoft/azure/provider.yaml | 1 + .../providers/microsoft/mssql/provider.yaml | 1 + .../providers/microsoft/psrp/provider.yaml | 1 + .../providers/microsoft/winrm/provider.yaml | 1 + airflow/providers/mongo/provider.yaml | 1 + airflow/providers/mysql/provider.yaml | 1 + airflow/providers/neo4j/provider.yaml | 1 + airflow/providers/odbc/provider.yaml | 1 + airflow/providers/openai/provider.yaml | 1 + airflow/providers/openfaas/provider.yaml | 1 + airflow/providers/openlineage/provider.yaml | 1 + airflow/providers/opensearch/provider.yaml | 1 + airflow/providers/opsgenie/provider.yaml | 1 + airflow/providers/oracle/provider.yaml | 1 + airflow/providers/pagerduty/provider.yaml | 1 + airflow/providers/papermill/provider.yaml | 1 + airflow/providers/pgvector/provider.yaml | 1 + airflow/providers/pinecone/provider.yaml | 1 + airflow/providers/plexus/provider.yaml | 1 + airflow/providers/postgres/provider.yaml | 1 + airflow/providers/presto/provider.yaml | 1 + airflow/providers/redis/provider.yaml | 1 + airflow/providers/salesforce/provider.yaml | 1 + airflow/providers/samba/provider.yaml | 1 + airflow/providers/segment/provider.yaml | 1 + airflow/providers/sendgrid/provider.yaml | 1 + airflow/providers/sftp/provider.yaml | 1 + airflow/providers/singularity/provider.yaml | 1 + airflow/providers/slack/provider.yaml | 1 + airflow/providers/smtp/provider.yaml | 1 + airflow/providers/snowflake/provider.yaml | 1 + airflow/providers/sqlite/provider.yaml | 1 + airflow/providers/ssh/provider.yaml | 1 + airflow/providers/tableau/provider.yaml | 1 + airflow/providers/tabular/provider.yaml | 1 + airflow/providers/telegram/provider.yaml | 1 + airflow/providers/trino/provider.yaml | 1 + airflow/providers/vertica/provider.yaml | 1 + airflow/providers/weaviate/provider.yaml | 1 + airflow/providers/yandex/provider.yaml | 1 + airflow/providers/zendesk/provider.yaml | 1 + .../commands/release_management_commands.py | 1 - .../provider_documentation.py | 33 +++++++++++---- .../prepare_providers/provider_packages.py | 16 +++---- .../src/airflow_breeze/utils/packages.py | 42 +++++++++++++------ dev/refresh_images.sh | 2 +- 97 files changed, 169 insertions(+), 33 deletions(-) diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml index 74667f0de15ad..f9164f357da02 100644 --- a/.github/actions/build-prod-images/action.yml +++ b/.github/actions/build-prod-images/action.yml @@ -46,9 +46,9 @@ runs: run: > breeze release-management prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 - - name: "Move dist packages to docker-context files" + - name: "Copy dist packages to docker-context files" shell: bash - run: mv -v ./dist/*.whl ./docker-context-files + run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files - name: "Download constraints from the CI build" uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a9ffb22515dc..ad5388664d620 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1930,8 +1930,8 @@ jobs: DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} COMMIT_SHA: ${{ github.sha }} if: matrix.platform == 'linux/amd64' - - name: "Move dist packages to docker-context files" - run: mv -v ./dist/*.whl ./docker-context-files + - name: "Copy dist packages to docker-context files" + run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" run: > breeze prod-image build diff --git a/airflow/provider.yaml.schema.json b/airflow/provider.yaml.schema.json index da32cd1c401e7..be589f5490fec 100644 --- a/airflow/provider.yaml.schema.json +++ b/airflow/provider.yaml.schema.json @@ -380,6 +380,16 @@ "type": "string" } } + }, + "source-date-epoch": { + "type": "integer", + "description": "Source date epoch - seconds since epoch (gmtime) when the release documentation was prepared. Used to generate reproducible package builds with flint.", + + "minimum": 0, + "default": 0, + "examples": [ + 1609459200 + ] } }, "additionalProperties": false, @@ -442,6 +452,7 @@ "package-name", "description", "suspended", + "source-date-epoch", "dependencies", "versions" ] diff --git a/airflow/providers/airbyte/provider.yaml b/airflow/providers/airbyte/provider.yaml index fb80c9872ebf4..7eabe2cfbdb39 100644 --- a/airflow/providers/airbyte/provider.yaml +++ b/airflow/providers/airbyte/provider.yaml @@ -22,6 +22,7 @@ description: | `Airbyte `__ suspended: false +source-date-epoch: 1700148043 versions: - 3.4.0 - 3.3.2 diff --git a/airflow/providers/alibaba/provider.yaml b/airflow/providers/alibaba/provider.yaml index 5a596fd31a877..2e0664e1b03cd 100644 --- a/airflow/providers/alibaba/provider.yaml +++ b/airflow/providers/alibaba/provider.yaml @@ -22,6 +22,7 @@ description: | Alibaba Cloud integration (including `Alibaba Cloud `__). suspended: false +source-date-epoch: 1700148044 versions: - 2.6.0 - 2.5.3 diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 3b357520a02b2..7873c40f31528 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -22,6 +22,7 @@ description: | Amazon integration (including `Amazon Web Services (AWS) `__). suspended: false +source-date-epoch: 1700148045 versions: - 8.11.0 - 8.10.0 diff --git a/airflow/providers/apache/beam/provider.yaml b/airflow/providers/apache/beam/provider.yaml index dc42d34e8c408..6910e281fa55a 100644 --- a/airflow/providers/apache/beam/provider.yaml +++ b/airflow/providers/apache/beam/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Beam `__. suspended: false +source-date-epoch: 1700148046 versions: - 5.3.0 - 5.2.3 diff --git a/airflow/providers/apache/cassandra/provider.yaml b/airflow/providers/apache/cassandra/provider.yaml index e430d345348ef..a427c959e8069 100644 --- a/airflow/providers/apache/cassandra/provider.yaml +++ b/airflow/providers/apache/cassandra/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Cassandra `__. suspended: false +source-date-epoch: 1700148046 versions: - 3.3.0 - 3.2.1 diff --git a/airflow/providers/apache/drill/provider.yaml b/airflow/providers/apache/drill/provider.yaml index fb9314c262a8d..c5b50280b41ae 100644 --- a/airflow/providers/apache/drill/provider.yaml +++ b/airflow/providers/apache/drill/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Drill `__. suspended: false +source-date-epoch: 1700148047 versions: - 2.5.0 - 2.4.4 diff --git a/airflow/providers/apache/druid/provider.yaml b/airflow/providers/apache/druid/provider.yaml index 0e982d67276b1..e9c46340ddf83 100644 --- a/airflow/providers/apache/druid/provider.yaml +++ b/airflow/providers/apache/druid/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Druid `__. suspended: false +source-date-epoch: 1700148047 versions: - 3.6.0 - 3.5.0 diff --git a/airflow/providers/apache/flink/provider.yaml b/airflow/providers/apache/flink/provider.yaml index 0d6cda3412eae..99c8041841f98 100644 --- a/airflow/providers/apache/flink/provider.yaml +++ b/airflow/providers/apache/flink/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Flink `__ suspended: false +source-date-epoch: 1700148048 versions: - 1.2.0 - 1.1.3 diff --git a/airflow/providers/apache/hdfs/provider.yaml b/airflow/providers/apache/hdfs/provider.yaml index bfa836c13456c..5a52e6ebb4b7a 100644 --- a/airflow/providers/apache/hdfs/provider.yaml +++ b/airflow/providers/apache/hdfs/provider.yaml @@ -23,6 +23,7 @@ description: | and `WebHDFS `__. suspended: false +source-date-epoch: 1700148048 versions: - 4.2.0 - 4.1.1 diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index 82e8f1e5e0652..4b2dbf3958998 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Hive `__ suspended: false +source-date-epoch: 1700148049 versions: - 6.2.0 - 6.1.6 diff --git a/airflow/providers/apache/impala/provider.yaml b/airflow/providers/apache/impala/provider.yaml index 77a6ec492464f..a997f3053f79d 100644 --- a/airflow/providers/apache/impala/provider.yaml +++ b/airflow/providers/apache/impala/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Impala `__. suspended: false +source-date-epoch: 1700148050 versions: - 1.2.0 - 1.1.3 diff --git a/airflow/providers/apache/kafka/provider.yaml b/airflow/providers/apache/kafka/provider.yaml index ab8ad48f04a24..674b48380c4e1 100644 --- a/airflow/providers/apache/kafka/provider.yaml +++ b/airflow/providers/apache/kafka/provider.yaml @@ -20,6 +20,7 @@ package-name: apache-airflow-providers-apache-kafka name: Apache Kafka suspended: false +source-date-epoch: 1700148050 description: | `Apache Kafka `__ versions: diff --git a/airflow/providers/apache/kylin/provider.yaml b/airflow/providers/apache/kylin/provider.yaml index 07af03e1c6015..fe36480fa3c1a 100644 --- a/airflow/providers/apache/kylin/provider.yaml +++ b/airflow/providers/apache/kylin/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Kylin `__ suspended: false +source-date-epoch: 1700148051 versions: - 3.3.0 - 3.2.1 diff --git a/airflow/providers/apache/livy/provider.yaml b/airflow/providers/apache/livy/provider.yaml index 85f1d03973868..b62796e54753c 100644 --- a/airflow/providers/apache/livy/provider.yaml +++ b/airflow/providers/apache/livy/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Livy `__ suspended: false +source-date-epoch: 1700148051 versions: - 3.6.0 - 3.5.4 diff --git a/airflow/providers/apache/pig/provider.yaml b/airflow/providers/apache/pig/provider.yaml index e2f3571320b57..0dbe8af52af4e 100644 --- a/airflow/providers/apache/pig/provider.yaml +++ b/airflow/providers/apache/pig/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Pig `__ suspended: false +source-date-epoch: 1700148052 versions: - 4.2.0 - 4.1.2 diff --git a/airflow/providers/apache/pinot/provider.yaml b/airflow/providers/apache/pinot/provider.yaml index 347c5c3d59930..23c6e8a3fab08 100644 --- a/airflow/providers/apache/pinot/provider.yaml +++ b/airflow/providers/apache/pinot/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Pinot `__ suspended: false +source-date-epoch: 1700148052 versions: - 4.2.0 - 4.1.4 diff --git a/airflow/providers/apache/spark/provider.yaml b/airflow/providers/apache/spark/provider.yaml index 31f2728a5c1b8..d819d56a3888b 100644 --- a/airflow/providers/apache/spark/provider.yaml +++ b/airflow/providers/apache/spark/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Spark `__ suspended: false +source-date-epoch: 1700148053 versions: - 4.4.0 - 4.3.0 diff --git a/airflow/providers/apache/sqoop/provider.yaml b/airflow/providers/apache/sqoop/provider.yaml index f959f50cb3b67..80934eb58704c 100644 --- a/airflow/providers/apache/sqoop/provider.yaml +++ b/airflow/providers/apache/sqoop/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Sqoop `__ suspended: false +source-date-epoch: 1700148053 versions: - 4.1.0 - 4.0.0 diff --git a/airflow/providers/apprise/provider.yaml b/airflow/providers/apprise/provider.yaml index 90f3cbcdb4a78..248a546bb0698 100644 --- a/airflow/providers/apprise/provider.yaml +++ b/airflow/providers/apprise/provider.yaml @@ -24,6 +24,7 @@ description: | `Apprise `__ suspended: false +source-date-epoch: 1700148054 versions: - 1.1.0 diff --git a/airflow/providers/arangodb/provider.yaml b/airflow/providers/arangodb/provider.yaml index 5e4b602b3f5e4..b5a83e7e90b95 100644 --- a/airflow/providers/arangodb/provider.yaml +++ b/airflow/providers/arangodb/provider.yaml @@ -26,6 +26,7 @@ dependencies: - python-arango>=7.3.2 suspended: false +source-date-epoch: 1700148054 versions: - 2.3.0 - 2.2.2 diff --git a/airflow/providers/asana/provider.yaml b/airflow/providers/asana/provider.yaml index 8c1b8a0430bee..4a8bd8777c41b 100644 --- a/airflow/providers/asana/provider.yaml +++ b/airflow/providers/asana/provider.yaml @@ -22,6 +22,7 @@ description: | `Asana `__ suspended: false +source-date-epoch: 1700148055 versions: - 2.3.0 - 2.2.2 diff --git a/airflow/providers/atlassian/jira/provider.yaml b/airflow/providers/atlassian/jira/provider.yaml index 6d3e1c9cc9f90..04d0a7a36d3b5 100644 --- a/airflow/providers/atlassian/jira/provider.yaml +++ b/airflow/providers/atlassian/jira/provider.yaml @@ -22,6 +22,7 @@ description: | `Atlassian Jira `__ suspended: false +source-date-epoch: 1700148055 versions: - 2.2.0 - 2.1.1 diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index ff9e767c9df83..74f63bb4e6f88 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -22,6 +22,7 @@ description: | `Celery `__ suspended: false +source-date-epoch: 1700148056 versions: - 3.4.1 - 3.4.0 diff --git a/airflow/providers/cloudant/provider.yaml b/airflow/providers/cloudant/provider.yaml index 32f79df6c13e1..11a99600d19b5 100644 --- a/airflow/providers/cloudant/provider.yaml +++ b/airflow/providers/cloudant/provider.yaml @@ -22,6 +22,7 @@ description: | `IBM Cloudant `__ suspended: false +source-date-epoch: 1700148056 versions: - 3.3.0 - 3.2.1 diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/airflow/providers/cncf/kubernetes/provider.yaml index 4ea8a840879bf..cf1c107843977 100644 --- a/airflow/providers/cncf/kubernetes/provider.yaml +++ b/airflow/providers/cncf/kubernetes/provider.yaml @@ -22,6 +22,7 @@ description: | `Kubernetes `__ suspended: false +source-date-epoch: 1700148057 versions: - 7.9.0 - 7.8.0 diff --git a/airflow/providers/cohere/provider.yaml b/airflow/providers/cohere/provider.yaml index 55500e0e760ff..8a6b12f73acf0 100644 --- a/airflow/providers/cohere/provider.yaml +++ b/airflow/providers/cohere/provider.yaml @@ -24,6 +24,7 @@ description: | `Cohere `__ suspended: false +source-date-epoch: 1700148058 versions: - 1.0.0 diff --git a/airflow/providers/common/io/provider.yaml b/airflow/providers/common/io/provider.yaml index 1e1ecb392e294..fdc8134090b2a 100644 --- a/airflow/providers/common/io/provider.yaml +++ b/airflow/providers/common/io/provider.yaml @@ -22,6 +22,7 @@ description: | ``Common IO Provider`` suspended: false +source-date-epoch: 1700148058 versions: - 1.0.1 - 1.0.0 diff --git a/airflow/providers/common/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml index 4dc1e703262a1..c44888925045f 100644 --- a/airflow/providers/common/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -22,6 +22,7 @@ description: | `Common SQL Provider `__ suspended: false +source-date-epoch: 1700148058 versions: - 1.8.1 - 1.8.0 diff --git a/airflow/providers/daskexecutor/provider.yaml b/airflow/providers/daskexecutor/provider.yaml index 76e67944b88e9..5516edf827e1f 100644 --- a/airflow/providers/daskexecutor/provider.yaml +++ b/airflow/providers/daskexecutor/provider.yaml @@ -22,6 +22,7 @@ description: | `Dask `__ suspended: false +source-date-epoch: 1700148059 versions: - 1.1.0 - 1.0.1 diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml index 2890561e3814b..81ce8ee1b8454 100644 --- a/airflow/providers/databricks/provider.yaml +++ b/airflow/providers/databricks/provider.yaml @@ -22,6 +22,7 @@ description: | `Databricks `__ suspended: false +source-date-epoch: 1700148060 versions: - 5.0.0 - 4.7.0 diff --git a/airflow/providers/datadog/provider.yaml b/airflow/providers/datadog/provider.yaml index 7286c718fec67..6a757e377e446 100644 --- a/airflow/providers/datadog/provider.yaml +++ b/airflow/providers/datadog/provider.yaml @@ -22,6 +22,7 @@ description: | `Datadog `__ suspended: false +source-date-epoch: 1700148060 versions: - 3.4.0 - 3.3.2 diff --git a/airflow/providers/dbt/cloud/provider.yaml b/airflow/providers/dbt/cloud/provider.yaml index 76d181253f855..acc74672c22b6 100644 --- a/airflow/providers/dbt/cloud/provider.yaml +++ b/airflow/providers/dbt/cloud/provider.yaml @@ -22,6 +22,7 @@ description: | `dbt Cloud `__ suspended: false +source-date-epoch: 1700148061 versions: - 3.4.0 - 3.3.0 diff --git a/airflow/providers/dingding/provider.yaml b/airflow/providers/dingding/provider.yaml index 17a92cac77080..7a418f95a67f6 100644 --- a/airflow/providers/dingding/provider.yaml +++ b/airflow/providers/dingding/provider.yaml @@ -22,6 +22,7 @@ description: | `DingTalk `__ suspended: false +source-date-epoch: 1700148061 versions: - 3.3.0 - 3.2.1 diff --git a/airflow/providers/discord/provider.yaml b/airflow/providers/discord/provider.yaml index dc91840d8a1c2..465067a78ca2f 100644 --- a/airflow/providers/discord/provider.yaml +++ b/airflow/providers/discord/provider.yaml @@ -22,6 +22,7 @@ description: | `Discord `__ suspended: false +source-date-epoch: 1700148062 versions: - 3.4.1 - 3.4.0 diff --git a/airflow/providers/docker/provider.yaml b/airflow/providers/docker/provider.yaml index d54f2db7aedbd..bbf4553fb2cfe 100644 --- a/airflow/providers/docker/provider.yaml +++ b/airflow/providers/docker/provider.yaml @@ -22,6 +22,7 @@ description: | `Docker `__ suspended: false +source-date-epoch: 1700148062 versions: - 3.8.1 - 3.8.0 diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml index dc7e7829c1173..8b7704892619f 100644 --- a/airflow/providers/elasticsearch/provider.yaml +++ b/airflow/providers/elasticsearch/provider.yaml @@ -22,6 +22,7 @@ description: | `Elasticsearch `__ suspended: false +source-date-epoch: 1700148063 versions: - 5.1.1 - 5.1.0 diff --git a/airflow/providers/exasol/provider.yaml b/airflow/providers/exasol/provider.yaml index 8314c8960a7fe..f193510a7eb1f 100644 --- a/airflow/providers/exasol/provider.yaml +++ b/airflow/providers/exasol/provider.yaml @@ -22,6 +22,7 @@ description: | `Exasol `__ suspended: false +source-date-epoch: 1700148064 versions: - 4.3.0 - 4.2.5 diff --git a/airflow/providers/facebook/provider.yaml b/airflow/providers/facebook/provider.yaml index 46aa33f244d47..a1427c9a6e076 100644 --- a/airflow/providers/facebook/provider.yaml +++ b/airflow/providers/facebook/provider.yaml @@ -22,6 +22,7 @@ description: | `Facebook Ads `__ suspended: false +source-date-epoch: 1700148064 versions: - 3.3.0 - 3.2.2 diff --git a/airflow/providers/ftp/provider.yaml b/airflow/providers/ftp/provider.yaml index 226caf7ca85f2..dcdeb06e7b7b1 100644 --- a/airflow/providers/ftp/provider.yaml +++ b/airflow/providers/ftp/provider.yaml @@ -22,6 +22,7 @@ description: | `File Transfer Protocol (FTP) `__ suspended: false +source-date-epoch: 1700148065 versions: - 3.6.1 - 3.6.0 diff --git a/airflow/providers/github/provider.yaml b/airflow/providers/github/provider.yaml index 962acdb06745b..5737413b192a8 100644 --- a/airflow/providers/github/provider.yaml +++ b/airflow/providers/github/provider.yaml @@ -29,6 +29,7 @@ dependencies: - PyGithub!=1.58 suspended: false +source-date-epoch: 1700148065 versions: - 2.4.0 - 2.3.2 diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 4287801d2b204..0f72165a3014e 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -29,6 +29,7 @@ description: | - `Google Workspace `__ (formerly Google Suite) suspended: false +source-date-epoch: 1700148066 versions: - 10.11.1 - 10.11.0 diff --git a/airflow/providers/grpc/provider.yaml b/airflow/providers/grpc/provider.yaml index e5f345174857c..9f6b19c3ca205 100644 --- a/airflow/providers/grpc/provider.yaml +++ b/airflow/providers/grpc/provider.yaml @@ -22,6 +22,7 @@ description: | `gRPC `__ suspended: false +source-date-epoch: 1700148067 versions: - 3.3.0 - 3.2.2 diff --git a/airflow/providers/hashicorp/provider.yaml b/airflow/providers/hashicorp/provider.yaml index 68c26ccb6fb44..81974af832119 100644 --- a/airflow/providers/hashicorp/provider.yaml +++ b/airflow/providers/hashicorp/provider.yaml @@ -22,6 +22,7 @@ description: | Hashicorp including `Hashicorp Vault `__ suspended: false +source-date-epoch: 1700148068 versions: - 3.5.0 - 3.4.3 diff --git a/airflow/providers/http/provider.yaml b/airflow/providers/http/provider.yaml index ab9ca16c06849..183ecf1351a61 100644 --- a/airflow/providers/http/provider.yaml +++ b/airflow/providers/http/provider.yaml @@ -22,6 +22,7 @@ description: | `Hypertext Transfer Protocol (HTTP) `__ suspended: false +source-date-epoch: 1700148068 versions: - 4.7.0 - 4.6.0 diff --git a/airflow/providers/imap/provider.yaml b/airflow/providers/imap/provider.yaml index eae2b360cb8bd..9701bd392ba5b 100644 --- a/airflow/providers/imap/provider.yaml +++ b/airflow/providers/imap/provider.yaml @@ -23,6 +23,7 @@ description: | `Internet Message Access Protocol (IMAP) `__ suspended: false +source-date-epoch: 1700148069 versions: - 3.4.0 - 3.3.2 diff --git a/airflow/providers/influxdb/provider.yaml b/airflow/providers/influxdb/provider.yaml index 0282c73931681..9a8f105afb1b5 100644 --- a/airflow/providers/influxdb/provider.yaml +++ b/airflow/providers/influxdb/provider.yaml @@ -29,6 +29,7 @@ dependencies: - requests>=2.26.0 suspended: false +source-date-epoch: 1700148070 versions: - 2.3.0 - 2.2.3 diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index 8948dfeb50270..4182086dd320b 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -22,6 +22,7 @@ description: | `Java Database Connectivity (JDBC) `__ suspended: false +source-date-epoch: 1700148070 versions: - 4.1.0 - 4.0.2 diff --git a/airflow/providers/jenkins/provider.yaml b/airflow/providers/jenkins/provider.yaml index 12538e796f5f2..0ed5868dbf6f9 100644 --- a/airflow/providers/jenkins/provider.yaml +++ b/airflow/providers/jenkins/provider.yaml @@ -22,6 +22,7 @@ description: | `Jenkins `__ suspended: false +source-date-epoch: 1700148071 versions: - 3.4.0 - 3.3.2 diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index 6679c8b4de408..5d8b449bed4d1 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -20,6 +20,7 @@ name: Microsoft Azure description: | `Microsoft Azure `__ suspended: false +source-date-epoch: 1700148071 versions: - 8.2.0 - 8.1.0 diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml index ef61b852d4966..f0aa626c252b7 100644 --- a/airflow/providers/microsoft/mssql/provider.yaml +++ b/airflow/providers/microsoft/mssql/provider.yaml @@ -22,6 +22,7 @@ description: | `Microsoft SQL Server (MSSQL) `__ suspended: false +source-date-epoch: 1700148072 versions: - 3.5.0 - 3.4.2 diff --git a/airflow/providers/microsoft/psrp/provider.yaml b/airflow/providers/microsoft/psrp/provider.yaml index bb283157a3779..a7dc6044331d4 100644 --- a/airflow/providers/microsoft/psrp/provider.yaml +++ b/airflow/providers/microsoft/psrp/provider.yaml @@ -24,6 +24,7 @@ description: | `__. suspended: false +source-date-epoch: 1700148073 versions: - 2.4.0 - 2.3.2 diff --git a/airflow/providers/microsoft/winrm/provider.yaml b/airflow/providers/microsoft/winrm/provider.yaml index c56b2b6dcfb68..7fbd5f8cc393a 100644 --- a/airflow/providers/microsoft/winrm/provider.yaml +++ b/airflow/providers/microsoft/winrm/provider.yaml @@ -22,6 +22,7 @@ description: | `Windows Remote Management (WinRM) `__ suspended: false +source-date-epoch: 1700148073 versions: - 3.3.0 - 3.2.2 diff --git a/airflow/providers/mongo/provider.yaml b/airflow/providers/mongo/provider.yaml index 53a441627c2e0..32834c5587b79 100644 --- a/airflow/providers/mongo/provider.yaml +++ b/airflow/providers/mongo/provider.yaml @@ -22,6 +22,7 @@ description: | `MongoDB `__ suspended: false +source-date-epoch: 1700148074 versions: - 3.4.0 - 3.3.0 diff --git a/airflow/providers/mysql/provider.yaml b/airflow/providers/mysql/provider.yaml index 7b73fef4f7b3b..b2a744b8e6fb3 100644 --- a/airflow/providers/mysql/provider.yaml +++ b/airflow/providers/mysql/provider.yaml @@ -22,6 +22,7 @@ description: | `MySQL `__ suspended: false +source-date-epoch: 1700148074 versions: - 5.4.0 - 5.3.1 diff --git a/airflow/providers/neo4j/provider.yaml b/airflow/providers/neo4j/provider.yaml index b00407208c445..c9f5d9457c3a2 100644 --- a/airflow/providers/neo4j/provider.yaml +++ b/airflow/providers/neo4j/provider.yaml @@ -22,6 +22,7 @@ description: | `Neo4j `__ suspended: false +source-date-epoch: 1700148075 versions: - 3.4.0 - 3.3.3 diff --git a/airflow/providers/odbc/provider.yaml b/airflow/providers/odbc/provider.yaml index 513d089b22841..21facc2fd089d 100644 --- a/airflow/providers/odbc/provider.yaml +++ b/airflow/providers/odbc/provider.yaml @@ -22,6 +22,7 @@ description: | `ODBC `__ suspended: false +source-date-epoch: 1700148076 versions: - 4.1.0 - 4.0.0 diff --git a/airflow/providers/openai/provider.yaml b/airflow/providers/openai/provider.yaml index ffb07de7bbe9c..8884108ca6515 100644 --- a/airflow/providers/openai/provider.yaml +++ b/airflow/providers/openai/provider.yaml @@ -24,6 +24,7 @@ description: | `OpenAI `__ suspended: false +source-date-epoch: 1700148076 versions: - 1.0.0 diff --git a/airflow/providers/openfaas/provider.yaml b/airflow/providers/openfaas/provider.yaml index 3eb521d163efe..3c6b73f514e4f 100644 --- a/airflow/providers/openfaas/provider.yaml +++ b/airflow/providers/openfaas/provider.yaml @@ -22,6 +22,7 @@ description: | `OpenFaaS `__ suspended: false +source-date-epoch: 1700148077 versions: - 3.3.0 - 3.2.1 diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml index 5dd878d5fd0fd..8fed707c37671 100644 --- a/airflow/providers/openlineage/provider.yaml +++ b/airflow/providers/openlineage/provider.yaml @@ -22,6 +22,7 @@ description: | `OpenLineage `__ suspended: false +source-date-epoch: 1700148077 versions: - 1.2.1 - 1.2.0 diff --git a/airflow/providers/opensearch/provider.yaml b/airflow/providers/opensearch/provider.yaml index 270d7cd2085dd..a0bddd79d5103 100644 --- a/airflow/providers/opensearch/provider.yaml +++ b/airflow/providers/opensearch/provider.yaml @@ -22,6 +22,7 @@ description: | `Opensearch `__ suspended: false +source-date-epoch: 1700148078 versions: - 1.0.0 diff --git a/airflow/providers/opsgenie/provider.yaml b/airflow/providers/opsgenie/provider.yaml index da89196f34714..d60f55f6d7d98 100644 --- a/airflow/providers/opsgenie/provider.yaml +++ b/airflow/providers/opsgenie/provider.yaml @@ -22,6 +22,7 @@ description: | `Opsgenie `__ suspended: false +source-date-epoch: 1700148078 versions: - 5.2.0 - 5.1.1 diff --git a/airflow/providers/oracle/provider.yaml b/airflow/providers/oracle/provider.yaml index 3befc752e686d..2a95c4138ab78 100644 --- a/airflow/providers/oracle/provider.yaml +++ b/airflow/providers/oracle/provider.yaml @@ -22,6 +22,7 @@ description: | `Oracle `__ suspended: false +source-date-epoch: 1700148079 versions: - 3.8.0 - 3.7.4 diff --git a/airflow/providers/pagerduty/provider.yaml b/airflow/providers/pagerduty/provider.yaml index 1a03a52deedc1..8d016282362e2 100644 --- a/airflow/providers/pagerduty/provider.yaml +++ b/airflow/providers/pagerduty/provider.yaml @@ -22,6 +22,7 @@ description: | `Pagerduty `__ suspended: false +source-date-epoch: 1700148079 versions: - 3.4.0 - 3.3.1 diff --git a/airflow/providers/papermill/provider.yaml b/airflow/providers/papermill/provider.yaml index c85fde16772f5..8a7155ef585f2 100644 --- a/airflow/providers/papermill/provider.yaml +++ b/airflow/providers/papermill/provider.yaml @@ -22,6 +22,7 @@ description: | `Papermill `__ suspended: false +source-date-epoch: 1700148080 versions: - 3.4.0 - 3.2.1 diff --git a/airflow/providers/pgvector/provider.yaml b/airflow/providers/pgvector/provider.yaml index 1fedfe141b745..37a6471994409 100644 --- a/airflow/providers/pgvector/provider.yaml +++ b/airflow/providers/pgvector/provider.yaml @@ -24,6 +24,7 @@ description: | `pgvector `__ suspended: false +source-date-epoch: 1700148080 versions: - 1.0.0 diff --git a/airflow/providers/pinecone/provider.yaml b/airflow/providers/pinecone/provider.yaml index f3050cfe229b6..853c223ae6a64 100644 --- a/airflow/providers/pinecone/provider.yaml +++ b/airflow/providers/pinecone/provider.yaml @@ -24,6 +24,7 @@ description: | `Pinecone `__ suspended: false +source-date-epoch: 1700148081 versions: - 1.0.0 diff --git a/airflow/providers/plexus/provider.yaml b/airflow/providers/plexus/provider.yaml index bb25339651f8d..70d2251e8fe47 100644 --- a/airflow/providers/plexus/provider.yaml +++ b/airflow/providers/plexus/provider.yaml @@ -22,6 +22,7 @@ description: | `Plexus `__ suspended: false +source-date-epoch: 1700148081 versions: - 3.3.0 - 3.2.2 diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 78e716bfca4bd..4afc8bc6955df 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -22,6 +22,7 @@ description: | `PostgreSQL `__ suspended: false +source-date-epoch: 1700148082 versions: - 5.8.0 - 5.7.1 diff --git a/airflow/providers/presto/provider.yaml b/airflow/providers/presto/provider.yaml index 107375a3c8b17..fa20f97a7dd4e 100644 --- a/airflow/providers/presto/provider.yaml +++ b/airflow/providers/presto/provider.yaml @@ -22,6 +22,7 @@ description: | `Presto `__ suspended: false +source-date-epoch: 1700148082 versions: - 5.2.1 - 5.2.0 diff --git a/airflow/providers/redis/provider.yaml b/airflow/providers/redis/provider.yaml index 486555ea54130..052fc17a18e35 100644 --- a/airflow/providers/redis/provider.yaml +++ b/airflow/providers/redis/provider.yaml @@ -22,6 +22,7 @@ description: | `Redis `__ suspended: false +source-date-epoch: 1700148083 versions: - 3.4.0 - 3.3.2 diff --git a/airflow/providers/salesforce/provider.yaml b/airflow/providers/salesforce/provider.yaml index a494a6dc5c6a4..e68d51ab0bf21 100644 --- a/airflow/providers/salesforce/provider.yaml +++ b/airflow/providers/salesforce/provider.yaml @@ -22,6 +22,7 @@ description: | `Salesforce `__ suspended: false +source-date-epoch: 1700148084 versions: - 5.5.1 - 5.5.0 diff --git a/airflow/providers/samba/provider.yaml b/airflow/providers/samba/provider.yaml index 27d673e8e360a..0822edb681cf7 100644 --- a/airflow/providers/samba/provider.yaml +++ b/airflow/providers/samba/provider.yaml @@ -22,6 +22,7 @@ description: | `Samba `__ suspended: false +source-date-epoch: 1700148084 versions: - 4.3.0 - 4.2.2 diff --git a/airflow/providers/segment/provider.yaml b/airflow/providers/segment/provider.yaml index 22b17d9c3367c..81e68982f6bb8 100644 --- a/airflow/providers/segment/provider.yaml +++ b/airflow/providers/segment/provider.yaml @@ -22,6 +22,7 @@ description: | `Segment `__ suspended: false +source-date-epoch: 1700148085 versions: - 3.3.0 - 3.2.1 diff --git a/airflow/providers/sendgrid/provider.yaml b/airflow/providers/sendgrid/provider.yaml index f1e7e90c8feb2..e897e7af8cdd9 100644 --- a/airflow/providers/sendgrid/provider.yaml +++ b/airflow/providers/sendgrid/provider.yaml @@ -26,6 +26,7 @@ dependencies: - sendgrid>=6.0.0 suspended: false +source-date-epoch: 1700148085 versions: - 3.3.0 - 3.2.2 diff --git a/airflow/providers/sftp/provider.yaml b/airflow/providers/sftp/provider.yaml index 073700ff354d9..af2be40c4aa7c 100644 --- a/airflow/providers/sftp/provider.yaml +++ b/airflow/providers/sftp/provider.yaml @@ -22,6 +22,7 @@ description: | `SSH File Transfer Protocol (SFTP) `__ suspended: false +source-date-epoch: 1700148086 versions: - 4.7.0 - 4.6.1 diff --git a/airflow/providers/singularity/provider.yaml b/airflow/providers/singularity/provider.yaml index 443ed1a98a7d4..52a83b896a03d 100644 --- a/airflow/providers/singularity/provider.yaml +++ b/airflow/providers/singularity/provider.yaml @@ -22,6 +22,7 @@ description: | `Singularity `__ suspended: false +source-date-epoch: 1700148086 versions: - 3.3.0 - 3.2.2 diff --git a/airflow/providers/slack/provider.yaml b/airflow/providers/slack/provider.yaml index e1283fde7b284..73b7dc3e4e9d1 100644 --- a/airflow/providers/slack/provider.yaml +++ b/airflow/providers/slack/provider.yaml @@ -25,6 +25,7 @@ description: | - `Slack Incoming Webhook `__ suspended: false +source-date-epoch: 1700148087 versions: - 8.4.0 - 8.3.0 diff --git a/airflow/providers/smtp/provider.yaml b/airflow/providers/smtp/provider.yaml index 8ff02eb9faa9e..096eeed141185 100644 --- a/airflow/providers/smtp/provider.yaml +++ b/airflow/providers/smtp/provider.yaml @@ -23,6 +23,7 @@ description: | `Simple Mail Transfer Protocol (SMTP) `__ suspended: false +source-date-epoch: 1700148088 versions: - 1.4.1 - 1.4.0 diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index bdc5b3ef079cb..026ec5c14d038 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -22,6 +22,7 @@ description: | `Snowflake `__ suspended: false +source-date-epoch: 1700148088 versions: - 5.1.1 - 5.1.0 diff --git a/airflow/providers/sqlite/provider.yaml b/airflow/providers/sqlite/provider.yaml index 6bde2aca1cd2f..4a20baf34239c 100644 --- a/airflow/providers/sqlite/provider.yaml +++ b/airflow/providers/sqlite/provider.yaml @@ -22,6 +22,7 @@ description: | `SQLite `__ suspended: false +source-date-epoch: 1700148089 versions: - 3.5.0 - 3.4.3 diff --git a/airflow/providers/ssh/provider.yaml b/airflow/providers/ssh/provider.yaml index 0944c7dda3375..0330f00b94732 100644 --- a/airflow/providers/ssh/provider.yaml +++ b/airflow/providers/ssh/provider.yaml @@ -22,6 +22,7 @@ description: | `Secure Shell (SSH) `__ suspended: false +source-date-epoch: 1700148090 versions: - 3.8.1 - 3.8.0 diff --git a/airflow/providers/tableau/provider.yaml b/airflow/providers/tableau/provider.yaml index ef591d060b6bb..8e456d3140311 100644 --- a/airflow/providers/tableau/provider.yaml +++ b/airflow/providers/tableau/provider.yaml @@ -22,6 +22,7 @@ description: | `Tableau `__ suspended: false +source-date-epoch: 1700148090 versions: - 4.3.0 - 4.2.2 diff --git a/airflow/providers/tabular/provider.yaml b/airflow/providers/tabular/provider.yaml index 267068b71d80e..1fc5dc57706f1 100644 --- a/airflow/providers/tabular/provider.yaml +++ b/airflow/providers/tabular/provider.yaml @@ -22,6 +22,7 @@ description: | `Tabular `__ suspended: false +source-date-epoch: 1700148091 versions: - 1.3.0 - 1.2.1 diff --git a/airflow/providers/telegram/provider.yaml b/airflow/providers/telegram/provider.yaml index 36c59720f9d51..8241dc46560ad 100644 --- a/airflow/providers/telegram/provider.yaml +++ b/airflow/providers/telegram/provider.yaml @@ -22,6 +22,7 @@ description: | `Telegram `__ suspended: false +source-date-epoch: 1700148091 versions: - 4.2.0 - 4.1.1 diff --git a/airflow/providers/trino/provider.yaml b/airflow/providers/trino/provider.yaml index 562304635a503..5ae593b9e1cb4 100644 --- a/airflow/providers/trino/provider.yaml +++ b/airflow/providers/trino/provider.yaml @@ -22,6 +22,7 @@ description: | `Trino `__ suspended: false +source-date-epoch: 1700148092 versions: - 5.4.0 - 5.3.1 diff --git a/airflow/providers/vertica/provider.yaml b/airflow/providers/vertica/provider.yaml index 09ef356ecdf63..f809a8a1bab78 100644 --- a/airflow/providers/vertica/provider.yaml +++ b/airflow/providers/vertica/provider.yaml @@ -22,6 +22,7 @@ description: | `Vertica `__ suspended: false +source-date-epoch: 1700148092 versions: - 3.6.0 - 3.5.2 diff --git a/airflow/providers/weaviate/provider.yaml b/airflow/providers/weaviate/provider.yaml index 94224031da41c..7301f5021c26a 100644 --- a/airflow/providers/weaviate/provider.yaml +++ b/airflow/providers/weaviate/provider.yaml @@ -24,6 +24,7 @@ description: | `Weaviate `__ suspended: false +source-date-epoch: 1700148093 versions: - 1.0.0 diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml index 8073992a25b33..c275924076141 100644 --- a/airflow/providers/yandex/provider.yaml +++ b/airflow/providers/yandex/provider.yaml @@ -21,6 +21,7 @@ name: Yandex description: | Yandex including `Yandex.Cloud `__ suspended: false +source-date-epoch: 1700148093 versions: - 3.6.0 - 3.5.0 diff --git a/airflow/providers/zendesk/provider.yaml b/airflow/providers/zendesk/provider.yaml index 04fa4423c9de3..25f03bd351978 100644 --- a/airflow/providers/zendesk/provider.yaml +++ b/airflow/providers/zendesk/provider.yaml @@ -22,6 +22,7 @@ description: | `Zendesk `__ suspended: false +source-date-epoch: 1700148094 versions: - 4.4.0 - 4.3.2 diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index cdc40645ab01f..fe3494541bdad 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -481,7 +481,6 @@ def prepare_provider_packages( cleanup_build_remnants(target_provider_root_sources_path) build_provider_package( provider_id=provider_id, - version_suffix=version_suffix_for_pypi, package_format=package_format, target_provider_root_sources_path=target_provider_root_sources_path, ) diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py index 3c0370f33e382..0193f9e0abf87 100644 --- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py +++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py @@ -28,6 +28,7 @@ from enum import Enum from pathlib import Path from shutil import copyfile +from time import time from typing import Any, NamedTuple import jinja2 @@ -42,8 +43,9 @@ ProviderPackageDetails, get_provider_details, get_provider_jinja_context, - get_provider_packages_metadata, get_source_package_path, + refresh_provider_metadata_from_yaml_file, + refresh_provider_metadata_with_provider_id, render_template, ) from airflow_breeze.utils.run_utils import run_command @@ -458,13 +460,29 @@ def _update_version_in_provider_yaml( original_text = provider_yaml_path.read_text() new_text = re.sub(r"versions:", f"versions:\n - {v}", original_text, 1) provider_yaml_path.write_text(new_text) - # IMPORTANT!!! Whenever we update provider.yaml files, we MUST clear cache for - # get_provider_packages_metadata function, because otherwise anything next will not use it - get_provider_packages_metadata.cache_clear() get_console().print(f"[special]Bumped version to {v}\n") return with_breaking_changes, maybe_with_new_features +def _update_source_date_epoch_in_provider_yaml( + provider_package_id: str, +) -> None: + """ + Updates source date epoch in provider yaml that then can be used to generate reproducible packages. + + :param provider_package_id: provider package + """ + provider_yaml_path = get_source_package_path(provider_package_id) / "provider.yaml" + original_text = provider_yaml_path.read_text() + source_date_epoch = int(time()) + new_text = re.sub( + r"source-date-epoch: [0-9]*", f"source-date-epoch: {source_date_epoch}", original_text, 1 + ) + provider_yaml_path.write_text(new_text) + refresh_provider_metadata_with_provider_id(provider_package_id) + get_console().print(f"[special]Updated source-date-epoch to {source_date_epoch}\n") + + def _verify_changelog_exists(package: str) -> Path: provider_details = get_provider_details(package) changelog_path = Path(provider_details.source_provider_package_path) / "CHANGELOG.rst" @@ -687,11 +705,14 @@ def update_release_notes( with_breaking_changes, maybe_with_new_features = _update_version_in_provider_yaml( provider_package_id=provider_package_id, type_of_change=type_of_change ) + _update_source_date_epoch_in_provider_yaml(provider_package_id) proceed, list_of_list_of_changes, changes_as_table = _get_all_changes_for_package( provider_package_id=provider_package_id, base_branch=base_branch, reapply_templates_only=reapply_templates_only, ) + else: + _update_source_date_epoch_in_provider_yaml(provider_package_id) provider_details = get_provider_details(provider_package_id) _verify_changelog_exists(provider_details.provider_id) jinja_context = get_provider_documentation_jinja_context( @@ -951,9 +972,7 @@ def _replace_min_airflow_version_in_provider_yaml( provider_yaml_txt, ) provider_yaml_path.write_text(provider_yaml_txt) - # IMPORTANT!!! Whenever we update provider.yaml files, we MUST clear cache for - # get_provider_packages_metadata function, because otherwise anything next will not use it - get_provider_packages_metadata.cache_clear() + refresh_provider_metadata_from_yaml_file(provider_yaml_path) def update_min_airflow_version( diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py index 5b004eb27d528..2f3b7eface50c 100644 --- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py +++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py @@ -22,7 +22,6 @@ import sys from pathlib import Path from shutil import copytree, rmtree -from stat import S_IRGRP, S_IROTH, S_IRUSR, S_IWUSR from typing import IO, Any from airflow_breeze.utils.console import get_console @@ -179,9 +178,7 @@ def cleanup_build_remnants(target_provider_root_sources_path: Path): get_console().print(f"[info]Cleaned remnants in {target_provider_root_sources_path}\n") -def build_provider_package( - provider_id: str, version_suffix: str, target_provider_root_sources_path: Path, package_format: str -): +def build_provider_package(provider_id: str, target_provider_root_sources_path: Path, package_format: str): get_console().print( f"\n[info]Building provider package: {provider_id} in format {package_format} in " f"{target_provider_root_sources_path}\n" @@ -190,7 +187,14 @@ def build_provider_package( if package_format != "both": command.extend(["--format", package_format]) try: - run_command(command, check=True, cwd=target_provider_root_sources_path) + run_command( + command, + check=True, + cwd=target_provider_root_sources_path, + env={ + "SOURCE_DATE_EPOCH": str(get_provider_details(provider_id).source_date_epoch), + }, + ) except subprocess.CalledProcessError as ex: get_console().print("[error]The command returned an error %s", ex) raise PrepareReleasePackageErrorBuildingPackageException() @@ -207,8 +211,6 @@ def move_built_packages_and_cleanup( # Shutil can move packages also between filesystems target_file = dist_folder / file.name target_file.unlink(missing_ok=True) - # Change ownership to group/other to read the file - file.chmod(S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) shutil.move(file.as_posix(), dist_folder.as_posix()) if skip_cleanup: diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index df2c4b1d51632..3d59643dbca3e 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -84,6 +84,7 @@ class PluginInfo(NamedTuple): class ProviderPackageDetails(NamedTuple): provider_id: str + source_date_epoch: int full_package_name: str pypi_package_name: str source_provider_package_path: Path @@ -131,27 +132,41 @@ def from_requirement(cls, requirement_string: str) -> PipRequirements: return cls(package=package, version_required=version_required.strip()) -@lru_cache +PROVIDER_METADATA: dict[str, dict[str, Any]] = {} + + +def refresh_provider_metadata_from_yaml_file(provider_yaml_path: Path): + import jsonschema + import yaml + + schema = _load_schema() + with open(provider_yaml_path) as yaml_file: + provider = yaml.safe_load(yaml_file) + try: + jsonschema.validate(provider, schema=schema) + except jsonschema.ValidationError: + raise Exception(f"Unable to parse: {provider_yaml_path}.") + PROVIDER_METADATA[get_short_package_name(provider["package-name"])] = provider + + +def refresh_provider_metadata_with_provider_id(provider_id: str): + provider_yaml_path = get_source_package_path(provider_id) / "provider.yaml" + refresh_provider_metadata_from_yaml_file(provider_yaml_path) + + def get_provider_packages_metadata() -> dict[str, dict[str, Any]]: """ Load all data from providers files :return: A list containing the contents of all provider.yaml files. """ - import jsonschema - import yaml - schema = _load_schema() - result: dict[str, dict[str, Any]] = {} + if PROVIDER_METADATA: + return PROVIDER_METADATA + for provider_yaml_path in get_provider_yaml_paths(): - with open(provider_yaml_path) as yaml_file: - provider = yaml.safe_load(yaml_file) - try: - jsonschema.validate(provider, schema=schema) - except jsonschema.ValidationError: - raise Exception(f"Unable to parse: {provider_yaml_path}.") - result[get_short_package_name(provider["package-name"])] = provider - return result + refresh_provider_metadata_from_yaml_file(provider_yaml_path) + return PROVIDER_METADATA def validate_provider_info_with_runtime_schema(provider_info: dict[str, Any]) -> None: @@ -440,6 +455,7 @@ def get_provider_details(provider_id: str) -> ProviderPackageDetails: ) return ProviderPackageDetails( provider_id=provider_id, + source_date_epoch=provider_info["source-date-epoch"], full_package_name=f"airflow.providers.{provider_id}", pypi_package_name=f"apache-airflow-providers-{provider_id.replace('.', '-')}", source_provider_package_path=get_source_package_path(provider_id), diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index 638e86930cc1f..2b11229aa06bc 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -42,7 +42,7 @@ breeze release-management prepare-provider-packages \ breeze release-management prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 -mv -v ./dist/*.whl ./docker-context-files +mv -v ./dist/*.whl ./docker-context-files && chmod a+r ./docker-context-files/* breeze prod-image build \ --builder airflow_cache \