From 3606793ea90c2cc0129f0e3fbf707d2949ce571d Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 8 Jan 2024 17:44:00 +0100 Subject: [PATCH] Bump min version for grpcio-status in spark provider Previously we limited grpcio minimum version to stop backtracking of `pip` from happening and we could not do it in the limits of spark provider, becaue some google dependencies used it and conflicted with it. This problem is now gone as we have newer versions of google dependencies and we can not only safely move it to spark provider but also bump it slightly higher to limit the amount of backtracking we need to do. Extracted from #36537 --- Dockerfile.ci | 5 +---- airflow/providers/apache/spark/provider.yaml | 2 +- generated/provider_dependencies.json | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index e6f9964d84f81..6c62a61344bec 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1117,14 +1117,11 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}" # force them on the main Airflow package. Currently we need no extra limits as PIP 23.1+ has much better # dependency resolution and we do not need to limit the versions of the dependencies # -# Without grpcio-status limit, pip gets into very long backtracking -# We should attempt to remove it in the future -# # Aiobotocore is limited for eager upgrade because it either causes a long backtracking or # conflict when we do not limit it. It seems that `pip` has a hard time figuring the right # combination of dependencies for aiobotocore, botocore, boto3 and s3fs together # -ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="grpcio-status>=1.55.0 aiobotocore>=2.5.4" +ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="aiobotocore>=2.5.4" ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ARG VERSION_SUFFIX_FOR_PYPI="" diff --git a/airflow/providers/apache/spark/provider.yaml b/airflow/providers/apache/spark/provider.yaml index c48f2a6c4ba73..38ab0fac8c4d5 100644 --- a/airflow/providers/apache/spark/provider.yaml +++ b/airflow/providers/apache/spark/provider.yaml @@ -55,7 +55,7 @@ versions: dependencies: - apache-airflow>=2.6.0 - pyspark - - grpcio-status + - grpcio-status>=1.59.0 integrations: - integration-name: Apache Spark diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 0366b3ebab8e1..aa4274e41edbd 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -206,7 +206,7 @@ "apache.spark": { "deps": [ "apache-airflow>=2.6.0", - "grpcio-status", + "grpcio-status>=1.59.0", "pyspark" ], "cross-providers-deps": [