From 3a4beb7835874379343cf2a7968b35c90ad1043f Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 28 Apr 2024 23:59:16 +0100 Subject: [PATCH 1/6] added logging_config,snapstart,ephemeral_storage parameters to aws lambdacreation operator --- .../amazon/aws/hooks/lambda_function.py | 10 ++++++ .../aws/operators/test_lambda_function.py | 32 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/airflow/providers/amazon/aws/hooks/lambda_function.py b/airflow/providers/amazon/aws/hooks/lambda_function.py index 771a01aeb7fd1..dfab3e7931678 100644 --- a/airflow/providers/amazon/aws/hooks/lambda_function.py +++ b/airflow/providers/amazon/aws/hooks/lambda_function.py @@ -106,6 +106,9 @@ def create_lambda( image_config: Any | None = None, code_signing_config_arn: str | None = None, architectures: list[str] | None = None, + ephemeral_storage: Any | None = None, + snap_start: Any | None = None, + logging_config: Any | None = None, ) -> dict: """ Create a Lambda function. @@ -151,6 +154,10 @@ def create_lambda( A code-signing configuration includes a set of signing profiles, which define the trusted publishers for this function. :param architectures: The instruction set architecture that the function supports. + :param ephemeral_storage: The size of the function’s /tmp directory in MB. + The default value is 512, but can be any whole number between 512 and 10,240 MB + :param snap_start: The function’s SnapStart setting + :param logging_config: The function’s Amazon CloudWatch Logs configuration settings """ if package_type == "Zip": if handler is None: @@ -181,6 +188,9 @@ def create_lambda( "ImageConfig": image_config, "CodeSigningConfigArn": code_signing_config_arn, "Architectures": architectures, + "EphemeralStorage": ephemeral_storage, + "SnapStart": snap_start, + "LoggingConfig": logging_config } return self.conn.create_function(**trim_none_values(create_function_args)) diff --git a/tests/providers/amazon/aws/operators/test_lambda_function.py b/tests/providers/amazon/aws/operators/test_lambda_function.py index 723f7584d9937..06990bd458c0e 100644 --- a/tests/providers/amazon/aws/operators/test_lambda_function.py +++ b/tests/providers/amazon/aws/operators/test_lambda_function.py @@ -125,6 +125,38 @@ def test_create_lambda_deferrable(self, _): with pytest.raises(TaskDeferred): operator.execute(None) + @mock.patch.object(LambdaHook, "create_lambda") + @mock.patch.object(LambdaHook, "conn") + @pytest.mark.parametrize( + "config", + [ + pytest.param({"architectures": ["arm64"], + "logging_config": { + "LogFormat": "Text", + "LogGroup": "/custom/log-group/" + } + }, id="with-config-parameters"), + ], + ) + def test_create_lambda_with_using_config_argument(self, + mock_hook_conn, + mock_hook_create_lambda, config): + operator = LambdaCreateFunctionOperator( + task_id="task_test", + function_name=FUNCTION_NAME, + role=ROLE_ARN, + code={ + "ImageUri": IMAGE_URI, + }, + config=config + ) + operator.execute(None) + + mock_hook_create_lambda.assert_called_once() + mock_hook_conn.get_waiter.assert_not_called() + assert operator.config.get("logging_config") == config.get("logging_config") + assert operator.config.get("architectures") == config.get("architectures") + class TestLambdaInvokeFunctionOperator: @pytest.mark.parametrize("payload", PAYLOADS) From a02df59cbef73ad02efb0c0cf7882367f2331009 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Mon, 29 Apr 2024 00:32:07 +0100 Subject: [PATCH 2/6] updated test case name --- .../amazon/aws/operators/test_lambda_function.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/providers/amazon/aws/operators/test_lambda_function.py b/tests/providers/amazon/aws/operators/test_lambda_function.py index 06990bd458c0e..3d41d21bf55bc 100644 --- a/tests/providers/amazon/aws/operators/test_lambda_function.py +++ b/tests/providers/amazon/aws/operators/test_lambda_function.py @@ -135,12 +135,12 @@ def test_create_lambda_deferrable(self, _): "LogFormat": "Text", "LogGroup": "/custom/log-group/" } - }, id="with-config-parameters"), + }, id="with-config-argument"), ], ) - def test_create_lambda_with_using_config_argument(self, - mock_hook_conn, - mock_hook_create_lambda, config): + def test_create_lambda_using_config_argument(self, + mock_hook_conn, + mock_hook_create_lambda, config): operator = LambdaCreateFunctionOperator( task_id="task_test", function_name=FUNCTION_NAME, From 86bb6f9bdb74a3f5677f523ccc50a66804e7698c Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Mon, 29 Apr 2024 06:51:43 +0100 Subject: [PATCH 3/6] updated test case for snap_start,ephemeral_storage parameters --- .../amazon/aws/operators/test_lambda_function.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/providers/amazon/aws/operators/test_lambda_function.py b/tests/providers/amazon/aws/operators/test_lambda_function.py index 3d41d21bf55bc..718718617fef3 100644 --- a/tests/providers/amazon/aws/operators/test_lambda_function.py +++ b/tests/providers/amazon/aws/operators/test_lambda_function.py @@ -134,6 +134,12 @@ def test_create_lambda_deferrable(self, _): "logging_config": { "LogFormat": "Text", "LogGroup": "/custom/log-group/" + }, + "snap_start": { + "ApplyOn": "PublishedVersions" + }, + "ephemeral_storage": { + "Size": 1024 } }, id="with-config-argument"), ], @@ -156,6 +162,8 @@ def test_create_lambda_using_config_argument(self, mock_hook_conn.get_waiter.assert_not_called() assert operator.config.get("logging_config") == config.get("logging_config") assert operator.config.get("architectures") == config.get("architectures") + assert operator.config.get("snap_start") == config.get("snap_start") + assert operator.config.get("ephemeral_storage") == config.get("ephemeral_storage") class TestLambdaInvokeFunctionOperator: From e5d3b244ad2182ba29b9746f39794023f205ddd3 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Mon, 29 Apr 2024 10:06:27 +0100 Subject: [PATCH 4/6] executed static checks --- .../amazon/aws/hooks/lambda_function.py | 8 +++--- .../auth_manager/cli_commands/user_command.py | 8 +++--- airflow/reproducible_build.yaml | 4 +-- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 4 +-- .../aws/operators/test_lambda_function.py | 27 ++++++++----------- 6 files changed, 23 insertions(+), 30 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/lambda_function.py b/airflow/providers/amazon/aws/hooks/lambda_function.py index dfab3e7931678..dea2a4272b2f1 100644 --- a/airflow/providers/amazon/aws/hooks/lambda_function.py +++ b/airflow/providers/amazon/aws/hooks/lambda_function.py @@ -154,10 +154,10 @@ def create_lambda( A code-signing configuration includes a set of signing profiles, which define the trusted publishers for this function. :param architectures: The instruction set architecture that the function supports. - :param ephemeral_storage: The size of the function’s /tmp directory in MB. + :param ephemeral_storage: The size of the function's /tmp directory in MB. The default value is 512, but can be any whole number between 512 and 10,240 MB - :param snap_start: The function’s SnapStart setting - :param logging_config: The function’s Amazon CloudWatch Logs configuration settings + :param snap_start: The function's SnapStart setting + :param logging_config: The function's Amazon CloudWatch Logs configuration settings """ if package_type == "Zip": if handler is None: @@ -190,7 +190,7 @@ def create_lambda( "Architectures": architectures, "EphemeralStorage": ephemeral_storage, "SnapStart": snap_start, - "LoggingConfig": logging_config + "LoggingConfig": logging_config, } return self.conn.create_function(**trim_none_values(create_function_args)) diff --git a/airflow/providers/fab/auth_manager/cli_commands/user_command.py b/airflow/providers/fab/auth_manager/cli_commands/user_command.py index 3050a9e250e58..8b6db12017876 100644 --- a/airflow/providers/fab/auth_manager/cli_commands/user_command.py +++ b/airflow/providers/fab/auth_manager/cli_commands/user_command.py @@ -212,10 +212,10 @@ def users_import(args): users_created, users_updated = _import_users(users_list) if users_created: - print("Created the following users:\n\t{}".format("\n\t".join(users_created))) + print(f"Created the following users:\n\t{'\\n\\t'.join(users_created)}") if users_updated: - print("Updated the following users:\n\t{}".format("\n\t".join(users_updated))) + print(f"Updated the following users:\n\t{'\\n\\t'.join(users_updated)}") def _import_users(users_list: list[dict[str, Any]]): @@ -231,9 +231,7 @@ def _import_users(users_list: list[dict[str, Any]]): msg.append(f"[Item {row_num}]") for key, value in failure.items(): msg.append(f"\t{key}: {value}") - raise SystemExit( - "Error: Input file didn't pass validation. See below:\n{}".format("\n".join(msg)) - ) + raise SystemExit(f"Error: Input file didn't pass validation. See below:\n{'\\n'.join(msg)}") for user in users_list: roles = [] diff --git a/airflow/reproducible_build.yaml b/airflow/reproducible_build.yaml index c6683aa2c04e4..84ae925af2f1b 100644 --- a/airflow/reproducible_build.yaml +++ b/airflow/reproducible_build.yaml @@ -1,2 +1,2 @@ -release-notes-hash: aad86522e49984ce17db1b8647cfb54a -source-date-epoch: 1714165337 +release-notes-hash: b9faa9895e2b44095e6ed66a11e0f838 +source-date-epoch: 1714378351 diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index cdcf039446dd5..ad684671e9515 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -77757e21aee500cb7fe7fd75e0f158633a0037d4d74e6f45eb14238f901ebacd \ No newline at end of file +f84bb14caa65d32bb752301c8c721d335602939bd35eaf70e886be21ce893eb1 \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index fb280ee0ea7fc..b80fcc4298b18 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -1421,7 +1421,7 @@ task_instance--xcom -1 +0..N 1 @@ -1442,7 +1442,7 @@ task_instance--xcom -0..N +1 1 diff --git a/tests/providers/amazon/aws/operators/test_lambda_function.py b/tests/providers/amazon/aws/operators/test_lambda_function.py index 718718617fef3..9c977ee883ed7 100644 --- a/tests/providers/amazon/aws/operators/test_lambda_function.py +++ b/tests/providers/amazon/aws/operators/test_lambda_function.py @@ -130,23 +130,18 @@ def test_create_lambda_deferrable(self, _): @pytest.mark.parametrize( "config", [ - pytest.param({"architectures": ["arm64"], - "logging_config": { - "LogFormat": "Text", - "LogGroup": "/custom/log-group/" - }, - "snap_start": { - "ApplyOn": "PublishedVersions" - }, - "ephemeral_storage": { - "Size": 1024 - } - }, id="with-config-argument"), + pytest.param( + { + "architectures": ["arm64"], + "logging_config": {"LogFormat": "Text", "LogGroup": "/custom/log-group/"}, + "snap_start": {"ApplyOn": "PublishedVersions"}, + "ephemeral_storage": {"Size": 1024}, + }, + id="with-config-argument", + ), ], ) - def test_create_lambda_using_config_argument(self, - mock_hook_conn, - mock_hook_create_lambda, config): + def test_create_lambda_using_config_argument(self, mock_hook_conn, mock_hook_create_lambda, config): operator = LambdaCreateFunctionOperator( task_id="task_test", function_name=FUNCTION_NAME, @@ -154,7 +149,7 @@ def test_create_lambda_using_config_argument(self, code={ "ImageUri": IMAGE_URI, }, - config=config + config=config, ) operator.execute(None) From d658fb52c61f09d518f8a0fcf5e10956c3d76ff6 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Mon, 29 Apr 2024 10:22:02 +0100 Subject: [PATCH 5/6] updated changes as per main branch --- .../fab/auth_manager/cli_commands/user_command.py | 8 +++++--- airflow/reproducible_build.yaml | 4 ++-- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/airflow/providers/fab/auth_manager/cli_commands/user_command.py b/airflow/providers/fab/auth_manager/cli_commands/user_command.py index 8b6db12017876..3050a9e250e58 100644 --- a/airflow/providers/fab/auth_manager/cli_commands/user_command.py +++ b/airflow/providers/fab/auth_manager/cli_commands/user_command.py @@ -212,10 +212,10 @@ def users_import(args): users_created, users_updated = _import_users(users_list) if users_created: - print(f"Created the following users:\n\t{'\\n\\t'.join(users_created)}") + print("Created the following users:\n\t{}".format("\n\t".join(users_created))) if users_updated: - print(f"Updated the following users:\n\t{'\\n\\t'.join(users_updated)}") + print("Updated the following users:\n\t{}".format("\n\t".join(users_updated))) def _import_users(users_list: list[dict[str, Any]]): @@ -231,7 +231,9 @@ def _import_users(users_list: list[dict[str, Any]]): msg.append(f"[Item {row_num}]") for key, value in failure.items(): msg.append(f"\t{key}: {value}") - raise SystemExit(f"Error: Input file didn't pass validation. See below:\n{'\\n'.join(msg)}") + raise SystemExit( + "Error: Input file didn't pass validation. See below:\n{}".format("\n".join(msg)) + ) for user in users_list: roles = [] diff --git a/airflow/reproducible_build.yaml b/airflow/reproducible_build.yaml index 84ae925af2f1b..c6683aa2c04e4 100644 --- a/airflow/reproducible_build.yaml +++ b/airflow/reproducible_build.yaml @@ -1,2 +1,2 @@ -release-notes-hash: b9faa9895e2b44095e6ed66a11e0f838 -source-date-epoch: 1714378351 +release-notes-hash: aad86522e49984ce17db1b8647cfb54a +source-date-epoch: 1714165337 diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index ad684671e9515..901cb4a60c856 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -f84bb14caa65d32bb752301c8c721d335602939bd35eaf70e886be21ce893eb1 \ No newline at end of file +77757e21aee500cb7fe7fd75e0f158633a0037d4d74e6f45eb14238f901ebacd diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index b80fcc4298b18..fb280ee0ea7fc 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -1421,7 +1421,7 @@ task_instance--xcom -0..N +1 1 @@ -1442,7 +1442,7 @@ task_instance--xcom -1 +0..N 1 From da5cd0e94a061c701a6590949598800360f17670 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Mon, 29 Apr 2024 15:34:18 +0100 Subject: [PATCH 6/6] removed newline from airflow_erd.sha256 file --- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 901cb4a60c856..cdcf039446dd5 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -77757e21aee500cb7fe7fd75e0f158633a0037d4d74e6f45eb14238f901ebacd +77757e21aee500cb7fe7fd75e0f158633a0037d4d74e6f45eb14238f901ebacd \ No newline at end of file