From 5a61a7869e72ed674b259465d61e5d12548f4e43 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 1 Jul 2025 12:19:11 +0200 Subject: [PATCH 1/2] Bump pyarrow to 16.1.0 minimum version for several providers (#52635) Pyarrow < 16.1.0 does not play well with numpy 2. Bumping it to 16.1.0 as minimum version should make compatibility tests to not downgrade to versions that are not compoatible when numpy 2 is already installed. It should also prevent our users from accidentally downgrading pyarrow or not upgrading it when numpy is upgraded to >= 2.0.0. (cherry picked from commit 87e34a69638e39c46697cd483249771ede793568) --- providers/apache/beam/pyproject.toml | 2 +- .../apache/beam/src/airflow/providers/apache/beam/__init__.py | 4 ++-- providers/databricks/pyproject.toml | 2 +- .../databricks/src/airflow/providers/databricks/__init__.py | 4 ++-- providers/google/pyproject.toml | 2 +- providers/snowflake/pyproject.toml | 2 +- .../snowflake/src/airflow/providers/snowflake/__init__.py | 4 ++-- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/providers/apache/beam/pyproject.toml b/providers/apache/beam/pyproject.toml index 951ab11d939e1..aefab025b2462 100644 --- a/providers/apache/beam/pyproject.toml +++ b/providers/apache/beam/pyproject.toml @@ -61,7 +61,7 @@ dependencies = [ # Apache Beam > 2.53.0 and pyarrow > 14.0.1 fix https://nvd.nist.gov/vuln/detail/CVE-2023-47248. 'apache-beam>=2.53.0; python_version < "3.12"', 'apache-beam>=2.57.0; python_version >= "3.12"', - "pyarrow>=14.0.1", + "pyarrow>=16.1.0", "numpy>=1.26.0", ] diff --git a/providers/apache/beam/src/airflow/providers/apache/beam/__init__.py b/providers/apache/beam/src/airflow/providers/apache/beam/__init__.py index 8b0417a318a79..81986f32590b0 100644 --- a/providers/apache/beam/src/airflow/providers/apache/beam/__init__.py +++ b/providers/apache/beam/src/airflow/providers/apache/beam/__init__.py @@ -32,8 +32,8 @@ __version__ = "6.0.4" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.9.0" + "2.10.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-apache-beam:{__version__}` needs Apache Airflow 2.9.0+" + f"The package `apache-airflow-providers-apache-beam:{__version__}` needs Apache Airflow 2.10.0+" ) diff --git a/providers/databricks/pyproject.toml b/providers/databricks/pyproject.toml index 8cd8b79be57cf..b39d9a13d4d4f 100644 --- a/providers/databricks/pyproject.toml +++ b/providers/databricks/pyproject.toml @@ -64,7 +64,7 @@ dependencies = [ "aiohttp>=3.9.2, <4", "mergedeep>=1.3.4", "pandas>=2.1.2,<2.2", - "pyarrow>=14.0.1", + "pyarrow>=16.1.0", ] # The optional dependencies should be modified in place in the generated file diff --git a/providers/databricks/src/airflow/providers/databricks/__init__.py b/providers/databricks/src/airflow/providers/databricks/__init__.py index 4253cc5c9f626..2ff85e645ff95 100644 --- a/providers/databricks/src/airflow/providers/databricks/__init__.py +++ b/providers/databricks/src/airflow/providers/databricks/__init__.py @@ -32,8 +32,8 @@ __version__ = "7.3.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.9.0" + "2.10.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-databricks:{__version__}` needs Apache Airflow 2.9.0+" + f"The package `apache-airflow-providers-databricks:{__version__}` needs Apache Airflow 2.10.0+" ) diff --git a/providers/google/pyproject.toml b/providers/google/pyproject.toml index 2d1038aa43da4..4599b87531628 100644 --- a/providers/google/pyproject.toml +++ b/providers/google/pyproject.toml @@ -138,7 +138,7 @@ dependencies = [ # further constrain it since older versions are buggy. "proto-plus>=1.19.6", # Used to write parquet files by BaseSqlToGCSOperator - "pyarrow>=14.0.1", + "pyarrow>=16.1.0", "python-slugify>=7.0.0", "PyOpenSSL>=23.0.0", "sqlalchemy-bigquery>=1.2.1", diff --git a/providers/snowflake/pyproject.toml b/providers/snowflake/pyproject.toml index 9b97443577702..51adcb46d2db9 100644 --- a/providers/snowflake/pyproject.toml +++ b/providers/snowflake/pyproject.toml @@ -65,7 +65,7 @@ dependencies = [ # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723 # In addition FAB also limit sqlalchemy to < 2.0 "pandas>=2.1.2,<2.2", - "pyarrow>=14.0.1", + "pyarrow>=16.1.0", "snowflake-connector-python>=3.7.1", "snowflake-sqlalchemy>=1.4.0", "snowflake-snowpark-python>=1.17.0;python_version<'3.12'", diff --git a/providers/snowflake/src/airflow/providers/snowflake/__init__.py b/providers/snowflake/src/airflow/providers/snowflake/__init__.py index c747051c604a8..1370df61f23fc 100644 --- a/providers/snowflake/src/airflow/providers/snowflake/__init__.py +++ b/providers/snowflake/src/airflow/providers/snowflake/__init__.py @@ -32,8 +32,8 @@ __version__ = "6.2.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( - "2.9.0" + "2.10.0" ): raise RuntimeError( - f"The package `apache-airflow-providers-snowflake:{__version__}` needs Apache Airflow 2.9.0+" + f"The package `apache-airflow-providers-snowflake:{__version__}` needs Apache Airflow 2.10.0+" ) From f1bbb7c97bea5fcf9914efae99e3136a91dfb006 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 1 Jul 2025 12:49:55 +0200 Subject: [PATCH 2/2] [v3-0-test] Bump pyarrow to 16.1.0 minimum version for several providers (#52635) Pyarrow < 16.1.0 does not play well with numpy 2. Bumping it to 16.1.0 as minimum version should make compatibility tests to not downgrade to versions that are not compoatible when numpy 2 is already installed. It should also prevent our users from accidentally downgrading pyarrow or not upgrading it when numpy is upgraded to >= 2.0.0. (cherry picked from commit 87e34a69638e39c46697cd483249771ede793568) Co-authored-by: Jarek Potiuk