From 07f451a1b36c6bd8d7baa2340d5da09569168dc4 Mon Sep 17 00:00:00 2001 From: Josh Dimarsky <24758845+yehoshuadimarsky@users.noreply.github.com> Date: Fri, 31 May 2024 15:13:45 -0400 Subject: [PATCH 1/5] bump minimum boto3 1.34.52 for AWS provider --- airflow/providers/amazon/provider.yaml | 4 ++-- generated/provider_dependencies.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 41aba21c36b10..3bc319a000114 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -92,8 +92,8 @@ dependencies: # We should update minimum version of boto3 and here regularly to avoid `pip` backtracking with the number # of candidates to consider. Make sure to configure boto3 version here as well as in all the tools below # in the `devel-dependencies` section to be the same minimum version. - - boto3>=1.33.0 - - botocore>=1.33.0 + - boto3>=1.34.52 + - botocore>=1.34.52 - inflection>=0.5.1 # Allow a wider range of watchtower versions for flexibility among users - watchtower>=2.0.1,<4 diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 71c1f13cdb239..d163d8d784ed9 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -32,8 +32,8 @@ "apache-airflow-providers-http", "apache-airflow>=2.7.0", "asgiref", - "boto3>=1.33.0", - "botocore>=1.33.0", + "boto3>=1.34.52", + "botocore>=1.34.52", "inflection>=0.5.1", "jmespath", "jsonpath_ng>=1.5.3", From f2adb9d63d549512511a299a4ab045dcfb076d7e Mon Sep 17 00:00:00 2001 From: Josh Dimarsky <24758845+yehoshuadimarsky@users.noreply.github.com> Date: Fri, 31 May 2024 15:18:12 -0400 Subject: [PATCH 2/5] update some deps --- airflow/providers/amazon/provider.yaml | 4 ++-- generated/provider_dependencies.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 3bc319a000114..742a37d845adb 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -133,9 +133,9 @@ devel-dependencies: - aws_xray_sdk>=2.12.0 - moto[cloudformation,glue]>=5.0.0 - mypy-boto3-appflow>=1.33.0 - - mypy-boto3-rds>=1.33.0 + - mypy-boto3-rds>=1.34.52 - mypy-boto3-redshift-data>=1.33.0 - - mypy-boto3-s3>=1.33.0 + - mypy-boto3-s3>=1.34.52 - s3fs>=2023.10.0 - openapi-schema-validator>=0.6.2 - openapi-spec-validator>=0.7.1 diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index d163d8d784ed9..935202d325710 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -46,9 +46,9 @@ "aws_xray_sdk>=2.12.0", "moto[cloudformation,glue]>=5.0.0", "mypy-boto3-appflow>=1.33.0", - "mypy-boto3-rds>=1.33.0", + "mypy-boto3-rds>=1.34.52", "mypy-boto3-redshift-data>=1.33.0", - "mypy-boto3-s3>=1.33.0", + "mypy-boto3-s3>=1.34.52", "openapi-schema-validator>=0.6.2", "openapi-spec-validator>=0.7.1", "s3fs>=2023.10.0" From 07c9e3ba63e138720d5291180599f9d29b4c94c1 Mon Sep 17 00:00:00 2001 From: Josh Dimarsky <24758845+yehoshuadimarsky@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:52:22 -0400 Subject: [PATCH 3/5] Doc-only change: AWS Batch operator requires boto3 1.34.52+ for 'ecs_properties_override' --- airflow/providers/amazon/aws/operators/batch.py | 1 + airflow/providers/amazon/provider.yaml | 8 ++++---- generated/provider_dependencies.json | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/airflow/providers/amazon/aws/operators/batch.py b/airflow/providers/amazon/aws/operators/batch.py index 849fc193461a5..bc2750c441115 100644 --- a/airflow/providers/amazon/aws/operators/batch.py +++ b/airflow/providers/amazon/aws/operators/batch.py @@ -66,6 +66,7 @@ class BatchOperator(BaseOperator): :param overrides: DEPRECATED, use container_overrides instead with the same value. :param container_overrides: the `containerOverrides` parameter for boto3 (templated) :param ecs_properties_override: the `ecsPropertiesOverride` parameter for boto3 (templated) + **NOTE** This requires `boto3` version 1.34.52+ :param node_overrides: the `nodeOverrides` parameter for boto3 (templated) :param share_identifier: The share identifier for the job. Don't specify this parameter if the job queue doesn't have a scheduling policy. diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 742a37d845adb..41aba21c36b10 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -92,8 +92,8 @@ dependencies: # We should update minimum version of boto3 and here regularly to avoid `pip` backtracking with the number # of candidates to consider. Make sure to configure boto3 version here as well as in all the tools below # in the `devel-dependencies` section to be the same minimum version. - - boto3>=1.34.52 - - botocore>=1.34.52 + - boto3>=1.33.0 + - botocore>=1.33.0 - inflection>=0.5.1 # Allow a wider range of watchtower versions for flexibility among users - watchtower>=2.0.1,<4 @@ -133,9 +133,9 @@ devel-dependencies: - aws_xray_sdk>=2.12.0 - moto[cloudformation,glue]>=5.0.0 - mypy-boto3-appflow>=1.33.0 - - mypy-boto3-rds>=1.34.52 + - mypy-boto3-rds>=1.33.0 - mypy-boto3-redshift-data>=1.33.0 - - mypy-boto3-s3>=1.34.52 + - mypy-boto3-s3>=1.33.0 - s3fs>=2023.10.0 - openapi-schema-validator>=0.6.2 - openapi-spec-validator>=0.7.1 diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 935202d325710..71c1f13cdb239 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -32,8 +32,8 @@ "apache-airflow-providers-http", "apache-airflow>=2.7.0", "asgiref", - "boto3>=1.34.52", - "botocore>=1.34.52", + "boto3>=1.33.0", + "botocore>=1.33.0", "inflection>=0.5.1", "jmespath", "jsonpath_ng>=1.5.3", @@ -46,9 +46,9 @@ "aws_xray_sdk>=2.12.0", "moto[cloudformation,glue]>=5.0.0", "mypy-boto3-appflow>=1.33.0", - "mypy-boto3-rds>=1.34.52", + "mypy-boto3-rds>=1.33.0", "mypy-boto3-redshift-data>=1.33.0", - "mypy-boto3-s3>=1.34.52", + "mypy-boto3-s3>=1.33.0", "openapi-schema-validator>=0.6.2", "openapi-spec-validator>=0.7.1", "s3fs>=2023.10.0" From 122f12b6e435349254d415981ec7d20f26f985b5 Mon Sep 17 00:00:00 2001 From: Josh Dimarsky <24758845+yehoshuadimarsky@users.noreply.github.com> Date: Tue, 4 Jun 2024 11:56:30 -0400 Subject: [PATCH 4/5] warn if ecsPropertiesOverride not recognized in Batch API call --- .../providers/amazon/aws/operators/batch.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/operators/batch.py b/airflow/providers/amazon/aws/operators/batch.py index bc2750c441115..4d2265b0a3897 100644 --- a/airflow/providers/amazon/aws/operators/batch.py +++ b/airflow/providers/amazon/aws/operators/batch.py @@ -30,6 +30,8 @@ from functools import cached_property from typing import TYPE_CHECKING, Any, Sequence +import botocore + from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import BaseOperator @@ -45,7 +47,11 @@ BatchCreateComputeEnvironmentTrigger, BatchJobTrigger, ) -from airflow.providers.amazon.aws.utils import trim_none_values, validate_execute_complete_event +from airflow.providers.amazon.aws.utils import ( + get_botocore_version, + trim_none_values, + validate_execute_complete_event, +) from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher from airflow.utils.types import NOTSET @@ -324,6 +330,17 @@ def submit_job(self, context: Context): try: response = self.hook.client.submit_job(**trim_none_values(args)) + except botocore.exceptions.ParamValidationError as error: + if ( + 'Unknown parameter in input: "ecsPropertiesOverride"' in str(error) + ) and self.ecs_properties_override: + self.log.error( + "You are attempting to use ecsPropertiesOverride and the Boto3 API returned an " + "error message which may indicate the need to update botocore to do this. \n" + "Support for using ecsPropertiesOverride was added in botocore 1.34.52 and you are using botocore %s", + ".".join(map(str, get_botocore_version())), + ) + raise except Exception as e: self.log.error( "AWS Batch job failed submission - job definition: %s - on queue %s", From dbeb95a625f7ea5d80c545133fa1f660b63aebac Mon Sep 17 00:00:00 2001 From: Josh Dimarsky <24758845+yehoshuadimarsky@users.noreply.github.com> Date: Tue, 4 Jun 2024 12:07:44 -0400 Subject: [PATCH 5/5] tweak message --- airflow/providers/amazon/aws/operators/batch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/operators/batch.py b/airflow/providers/amazon/aws/operators/batch.py index 4d2265b0a3897..6125c3714079b 100644 --- a/airflow/providers/amazon/aws/operators/batch.py +++ b/airflow/providers/amazon/aws/operators/batch.py @@ -335,7 +335,7 @@ def submit_job(self, context: Context): 'Unknown parameter in input: "ecsPropertiesOverride"' in str(error) ) and self.ecs_properties_override: self.log.error( - "You are attempting to use ecsPropertiesOverride and the Boto3 API returned an " + "You are attempting to use ecsPropertiesOverride and the botocore API returned an " "error message which may indicate the need to update botocore to do this. \n" "Support for using ecsPropertiesOverride was added in botocore 1.34.52 and you are using botocore %s", ".".join(map(str, get_botocore_version())),