From 6ee27b670af3448b51b77fc3f2771655fe83c9c9 Mon Sep 17 00:00:00 2001 From: phi Date: Fri, 9 Aug 2024 14:20:26 +0900 Subject: [PATCH 01/10] feat: first try --- airflow/providers/docker/decorators/docker.py | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/airflow/providers/docker/decorators/docker.py b/airflow/providers/docker/decorators/docker.py index d851c98aca5d9..6a6b575b2db00 100644 --- a/airflow/providers/docker/decorators/docker.py +++ b/airflow/providers/docker/decorators/docker.py @@ -18,13 +18,13 @@ import base64 import os -import pickle +import warnings from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, Callable, Sequence -import dill - from airflow.decorators.base import DecoratedOperator, task_decorator_factory +from airflow.exceptions import AirflowException, RemovedInAirflow3Warning +from airflow.operators.python import _SERIALIZERS, _SerializerTypeDef from airflow.providers.docker.operators.docker import DockerOperator from airflow.utils.python_virtualenv import write_python_script @@ -53,7 +53,6 @@ class _DockerDecoratedOperator(DecoratedOperator, DockerOperator): :param python_callable: A reference to an object that is callable :param python: Python binary name to use - :param use_dill: Whether dill should be used to serialize the callable :param expect_airflow: whether to expect airflow to be installed in the docker environment. if this one is specified, the script to run callable will attempt to load Airflow macros. :param op_kwargs: a dictionary of keyword arguments that will get unpacked @@ -63,6 +62,16 @@ class _DockerDecoratedOperator(DecoratedOperator, DockerOperator): :param multiple_outputs: if set, function return value will be unrolled to multiple XCom values. Dict will unroll to xcom values with keys as keys. Defaults to False. + :param serializer: Which serializer use to serialize the args and result. It can be one of the following: + + - ``"pickle"``: (default) Use pickle for serialization. Included in the Python Standard Library. + - ``"cloudpickle"``: Use cloudpickle for serialize more complex types, + this requires to include cloudpickle in your requirements. + - ``"dill"``: Use dill for serialize more complex types, + this requires to include dill in your requirements. + :param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize + the args and result (pickle is default). This allows more complex types + but requires you to include dill in your requirements. """ custom_operator_name = "@task.docker" @@ -74,12 +83,35 @@ def __init__( use_dill=False, python_command="python3", expect_airflow: bool = True, + serializer: _SerializerTypeDef | None = None, **kwargs, ) -> None: + if use_dill: + warnings.warn( + "`use_dill` is deprecated and will be removed in a future version. " + "Please provide serializer='dill' instead.", + RemovedInAirflow3Warning, + stacklevel=3, + ) + if serializer: + raise AirflowException( + "Both 'use_dill' and 'serializer' parameters are set. Please set only one of them" + ) + serializer = "dill" + serializer = serializer or "pickle" + if serializer not in _SERIALIZERS: + msg = ( + f"Unsupported serializer {serializer!r}. " + f"Expected one of {', '.join(map(repr, _SERIALIZERS))}" + ) + raise AirflowException(msg) + command = "placeholder command" self.python_command = python_command self.expect_airflow = expect_airflow - self.use_dill = use_dill + self.use_dill = serializer == "dill" + self.serializer: _SerializerTypeDef = serializer + super().__init__( command=command, retrieve_output=True, retrieve_output_path="/tmp/script.out", **kwargs ) @@ -128,9 +160,7 @@ def execute(self, context: Context): @property def pickling_library(self): - if self.use_dill: - return dill - return pickle + return _SERIALIZERS[self.serializer] def docker_task( From fe6d75475e9e2b0bed8392d221677afc8af23649 Mon Sep 17 00:00:00 2001 From: phi Date: Fri, 9 Aug 2024 20:40:30 +0900 Subject: [PATCH 02/10] fix: docker deco stub --- airflow/decorators/__init__.pyi | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/airflow/decorators/__init__.pyi b/airflow/decorators/__init__.pyi index 089e453d02b43..7bd0b7da7fad0 100644 --- a/airflow/decorators/__init__.pyi +++ b/airflow/decorators/__init__.pyi @@ -63,6 +63,8 @@ __all__ = [ _T = TypeVar("_T", bound=Task[..., Any] | _TaskDecorator[..., Any, Any]) +Serializer = Literal["pickle", "cloudpickle", "dill"] + class TaskDecoratorCollection: @overload def python( # type: ignore[misc] @@ -114,7 +116,7 @@ class TaskDecoratorCollection: # _PythonVirtualenvDecoratedOperator. requirements: None | Iterable[str] | str = None, python_version: None | str | int | float = None, - serializer: Literal["pickle", "cloudpickle", "dill"] | None = None, + serializer: Serializer | None = None, system_site_packages: bool = True, templates_dict: Mapping[str, Any] | None = None, pip_install_options: list[str] | None = None, @@ -188,7 +190,7 @@ class TaskDecoratorCollection: multiple_outputs: bool | None = None, # 'python_callable', 'op_args' and 'op_kwargs' since they are filled by # _PythonVirtualenvDecoratedOperator. - serializer: Literal["pickle", "cloudpickle", "dill"] | None = None, + serializer: Serializer | None = None, templates_dict: Mapping[str, Any] | None = None, show_return_value_in_logs: bool = True, env_vars: dict[str, str] | None = None, @@ -253,7 +255,7 @@ class TaskDecoratorCollection: # _PythonVirtualenvDecoratedOperator. requirements: None | Iterable[str] | str = None, python_version: None | str | int | float = None, - serializer: Literal["pickle", "cloudpickle", "dill"] | None = None, + serializer: Serializer | None = None, system_site_packages: bool = True, templates_dict: Mapping[str, Any] | None = None, pip_install_options: list[str] | None = None, @@ -316,7 +318,7 @@ class TaskDecoratorCollection: multiple_outputs: bool | None = None, # 'python_callable', 'op_args' and 'op_kwargs' since they are filled by # _PythonVirtualenvDecoratedOperator. - serializer: Literal["pickle", "cloudpickle", "dill"] | None = None, + serializer: Serializer | None = None, templates_dict: Mapping[str, Any] | None = None, show_return_value_in_logs: bool = True, use_dill: bool = False, @@ -379,8 +381,9 @@ class TaskDecoratorCollection: self, *, multiple_outputs: bool | None = None, - use_dill: bool = False, # Added by _DockerDecoratedOperator. python_command: str = "python3", + serializer: Serializer | None = None, + use_dill: bool = False, # Added by _DockerDecoratedOperator. # 'command', 'retrieve_output', and 'retrieve_output_path' are filled by # _DockerDecoratedOperator. image: str, @@ -432,8 +435,17 @@ class TaskDecoratorCollection: :param multiple_outputs: If set, function return value will be unrolled to multiple XCom values. Dict will unroll to XCom values with keys as XCom keys. Defaults to False. - :param use_dill: Whether to use dill or pickle for serialization :param python_command: Python command for executing functions, Default: python3 + :param serializer: Which serializer use to serialize the args and result. It can be one of the following: + + - ``"pickle"``: (default) Use pickle for serialization. Included in the Python Standard Library. + - ``"cloudpickle"``: Use cloudpickle for serialize more complex types, + this requires to include cloudpickle in your requirements. + - ``"dill"``: Use dill for serialize more complex types, + this requires to include dill in your requirements. + :param use_dill: Deprecated, use ``serializer`` instead. Whether to use dill to serialize + the args and result (pickle is default). This allows more complex types + but requires you to include dill in your requirements. :param image: Docker image from which to create the container. If image tag is omitted, "latest" will be used. :param api_version: Remote API version. Set to ``auto`` to automatically From 281fe3da27f6b211fbefac6978f1656052e1e79a Mon Sep 17 00:00:00 2001 From: phi Date: Fri, 9 Aug 2024 20:42:27 +0900 Subject: [PATCH 03/10] fix: use_dill test --- .../docker/decorators/test_docker.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/providers/docker/decorators/test_docker.py b/tests/providers/docker/decorators/test_docker.py index 93db9f211b4db..ed45273d29884 100644 --- a/tests/providers/docker/decorators/test_docker.py +++ b/tests/providers/docker/decorators/test_docker.py @@ -17,6 +17,7 @@ from __future__ import annotations import logging +from importlib.util import find_spec from io import StringIO as StringBuffer import pytest @@ -32,6 +33,10 @@ DEFAULT_DATE = timezone.datetime(2021, 9, 1) +DILL_INSTALLED = find_spec("dill") is not None +DILL_MARKER = pytest.mark.skipif(not DILL_INSTALLED, reason="`dill` is not installed") +CLOUDPICKLE_INSTALLED = find_spec("cloudpickle") is not None +CLOUDPICKLE_MARKER = pytest.mark.skipif(not CLOUDPICKLE_INSTALLED, reason="`cloudpickle` is not installed") class TestDockerDecorator: @@ -207,13 +212,21 @@ def f(): assert teardown_task.is_teardown assert teardown_task.on_failure_fail_dagrun is on_failure_fail_dagrun - @pytest.mark.parametrize("use_dill", [True, False]) - def test_deepcopy_with_python_operator(self, dag_maker, use_dill): + @pytest.mark.parametrize( + "serializer", + [ + pytest.param("pickle", id="pickle"), + pytest.param("dill", marks=DILL_MARKER, id="dill"), + pytest.param("cloudpickle", marks=CLOUDPICKLE_MARKER, id="cloudpickle"), + pytest.param(None, id="default"), + ], + ) + def test_deepcopy_with_python_operator(self, dag_maker, serializer): import copy from airflow.providers.docker.decorators.docker import _DockerDecoratedOperator - @task.docker(image="python:3.9-slim", auto_remove="force", use_dill=use_dill) + @task.docker(image="python:3.9-slim", auto_remove="force", serializer=serializer) def f(): import logging @@ -247,6 +260,7 @@ def g(): assert isinstance(clone_of_docker_operator, _DockerDecoratedOperator) assert some_task.command == clone_of_docker_operator.command assert some_task.expect_airflow == clone_of_docker_operator.expect_airflow + assert some_task.serializer == clone_of_docker_operator.serializer assert some_task.use_dill == clone_of_docker_operator.use_dill assert some_task.pickling_library is clone_of_docker_operator.pickling_library From 20726b106e9a5d367d4f7c0a344a77fa7b207418 Mon Sep 17 00:00:00 2001 From: phi Date: Fri, 9 Aug 2024 20:53:02 +0900 Subject: [PATCH 04/10] tests: ambiguous_serializer --- .../docker/decorators/test_docker.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/providers/docker/decorators/test_docker.py b/tests/providers/docker/decorators/test_docker.py index ed45273d29884..3515e98ef46ad 100644 --- a/tests/providers/docker/decorators/test_docker.py +++ b/tests/providers/docker/decorators/test_docker.py @@ -23,7 +23,7 @@ import pytest from airflow.decorators import setup, task, teardown -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, RemovedInAirflow3Warning from airflow.models import TaskInstance from airflow.models.dag import DAG from airflow.utils import timezone @@ -331,3 +331,23 @@ def f(): assert 'with open(sys.argv[4], "w") as file:' not in log_content last_line_of_docker_operator_log = log_content.splitlines()[-1] assert "ValueError: This task is expected to fail" in last_line_of_docker_operator_log + + @pytest.mark.parametrize( + "serializer", + [ + pytest.param("pickle", id="pickle"), + pytest.param("dill", marks=DILL_MARKER, id="dill"), + pytest.param("cloudpickle", marks=CLOUDPICKLE_MARKER, id="cloudpickle"), + ], + ) + def test_ambiguous_serializer(self, dag_maker, serializer): + @task.docker(image="python:3.9-slim", auto_remove="force", use_dill=True, serializer=serializer) + def f(): + pass + + with dag_maker(): + with pytest.warns(RemovedInAirflow3Warning, match="`use_dill` is deprecated and will be removed"): + with pytest.raises( + AirflowException, match="Both 'use_dill' and 'serializer' parameters are set" + ): + f() From f62a2420d2de28c4b8c3741bae36810fb61dd2bc Mon Sep 17 00:00:00 2001 From: phi Date: Fri, 9 Aug 2024 20:55:31 +0900 Subject: [PATCH 05/10] tests: invalid_serializer --- tests/providers/docker/decorators/test_docker.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/providers/docker/decorators/test_docker.py b/tests/providers/docker/decorators/test_docker.py index 3515e98ef46ad..da0a5dbb743d3 100644 --- a/tests/providers/docker/decorators/test_docker.py +++ b/tests/providers/docker/decorators/test_docker.py @@ -351,3 +351,13 @@ def f(): AirflowException, match="Both 'use_dill' and 'serializer' parameters are set" ): f() + + def test_invalid_serializer(self, dag_maker): + @task.docker(image="python:3.9-slim", auto_remove="force", serializer="airflow") + def f(): + """Ensure dill is correctly installed.""" + import dill # noqa: F401 + + with dag_maker(): + with pytest.raises(AirflowException, match="Unsupported serializer 'airflow'"): + f() From 683f43276ac345dc5604709a986b0d18145cf80a Mon Sep 17 00:00:00 2001 From: phi Date: Fri, 9 Aug 2024 21:02:21 +0900 Subject: [PATCH 06/10] tests: more --- .../docker/decorators/test_docker.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/providers/docker/decorators/test_docker.py b/tests/providers/docker/decorators/test_docker.py index da0a5dbb743d3..90ca8e911e8c8 100644 --- a/tests/providers/docker/decorators/test_docker.py +++ b/tests/providers/docker/decorators/test_docker.py @@ -361,3 +361,64 @@ def f(): with dag_maker(): with pytest.raises(AirflowException, match="Unsupported serializer 'airflow'"): f() + + @pytest.mark.parametrize( + "serializer", + [ + pytest.param( + "dill", + marks=pytest.mark.skipif( + DILL_INSTALLED, reason="For this test case `dill` shouldn't be installed" + ), + id="dill", + ), + pytest.param( + "cloudpickle", + marks=pytest.mark.skipif( + CLOUDPICKLE_INSTALLED, reason="For this test case `cloudpickle` shouldn't be installed" + ), + id="cloudpickle", + ), + ], + ) + def test_advanced_serializer_not_installed(self, dag_maker, serializer, caplog): + """Test case for check raising an error if dill/cloudpickle is not installed.""" + + @task.docker(image="python:3.9-slim", auto_remove="force", serializer=serializer) + def f(): ... + + with dag_maker(): + with pytest.raises(ModuleNotFoundError): + f() + assert f"Unable to import `{serializer}` module." in caplog.text + + @CLOUDPICKLE_MARKER + def test_add_cloudpickle(self, dag_maker): + @task.docker(image="python:3.9-slim", auto_remove="force", serializer="cloudpickle") + def f(): + """Ensure cloudpickle is correctly installed.""" + import cloudpickle # noqa: F401 + + with dag_maker(): + f() + + @DILL_MARKER + def test_add_dill(self, dag_maker): + @task.docker(image="python:3.9-slim", auto_remove="force", serializer="dill") + def f(): + """Ensure dill is correctly installed.""" + import dill # noqa: F401 + + with dag_maker(): + f() + + @DILL_MARKER + def test_add_dill_use_dill(self, dag_maker): + @task.docker(image="python:3.9-slim", auto_remove="force", use_dill=True) + def f(): + """Ensure dill is correctly installed.""" + import dill # noqa: F401 + + with dag_maker(): + with pytest.warns(RemovedInAirflow3Warning, match="`use_dill` is deprecated and will be removed"): + f() From 5b871ff40b4a4dd0947c46047c5f29eb5f75e28b Mon Sep 17 00:00:00 2001 From: phi Date: Fri, 9 Aug 2024 21:07:00 +0900 Subject: [PATCH 07/10] fix: replace AirflowProviderDeprecationWarning --- airflow/providers/docker/decorators/docker.py | 4 ++-- tests/providers/docker/decorators/test_docker.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/airflow/providers/docker/decorators/docker.py b/airflow/providers/docker/decorators/docker.py index 6a6b575b2db00..be000da48efd9 100644 --- a/airflow/providers/docker/decorators/docker.py +++ b/airflow/providers/docker/decorators/docker.py @@ -23,7 +23,7 @@ from typing import TYPE_CHECKING, Callable, Sequence from airflow.decorators.base import DecoratedOperator, task_decorator_factory -from airflow.exceptions import AirflowException, RemovedInAirflow3Warning +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.operators.python import _SERIALIZERS, _SerializerTypeDef from airflow.providers.docker.operators.docker import DockerOperator from airflow.utils.python_virtualenv import write_python_script @@ -90,7 +90,7 @@ def __init__( warnings.warn( "`use_dill` is deprecated and will be removed in a future version. " "Please provide serializer='dill' instead.", - RemovedInAirflow3Warning, + AirflowProviderDeprecationWarning, stacklevel=3, ) if serializer: diff --git a/tests/providers/docker/decorators/test_docker.py b/tests/providers/docker/decorators/test_docker.py index 90ca8e911e8c8..42b1a514a3a9c 100644 --- a/tests/providers/docker/decorators/test_docker.py +++ b/tests/providers/docker/decorators/test_docker.py @@ -23,7 +23,7 @@ import pytest from airflow.decorators import setup, task, teardown -from airflow.exceptions import AirflowException, RemovedInAirflow3Warning +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models import TaskInstance from airflow.models.dag import DAG from airflow.utils import timezone @@ -346,7 +346,9 @@ def f(): pass with dag_maker(): - with pytest.warns(RemovedInAirflow3Warning, match="`use_dill` is deprecated and will be removed"): + with pytest.warns( + AirflowProviderDeprecationWarning, match="`use_dill` is deprecated and will be removed" + ): with pytest.raises( AirflowException, match="Both 'use_dill' and 'serializer' parameters are set" ): @@ -420,5 +422,7 @@ def f(): import dill # noqa: F401 with dag_maker(): - with pytest.warns(RemovedInAirflow3Warning, match="`use_dill` is deprecated and will be removed"): + with pytest.warns( + AirflowProviderDeprecationWarning, match="`use_dill` is deprecated and will be removed" + ): f() From cbaf8dabec42a9705289e5d4834df91ff60ada66 Mon Sep 17 00:00:00 2001 From: phi Date: Fri, 9 Aug 2024 21:10:19 +0900 Subject: [PATCH 08/10] fix: revert stub --- airflow/decorators/__init__.pyi | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/airflow/decorators/__init__.pyi b/airflow/decorators/__init__.pyi index 7bd0b7da7fad0..e6d247918395c 100644 --- a/airflow/decorators/__init__.pyi +++ b/airflow/decorators/__init__.pyi @@ -63,8 +63,6 @@ __all__ = [ _T = TypeVar("_T", bound=Task[..., Any] | _TaskDecorator[..., Any, Any]) -Serializer = Literal["pickle", "cloudpickle", "dill"] - class TaskDecoratorCollection: @overload def python( # type: ignore[misc] @@ -116,7 +114,7 @@ class TaskDecoratorCollection: # _PythonVirtualenvDecoratedOperator. requirements: None | Iterable[str] | str = None, python_version: None | str | int | float = None, - serializer: Serializer | None = None, + serializer: Literal["pickle", "cloudpickle", "dill"] | None = None, system_site_packages: bool = True, templates_dict: Mapping[str, Any] | None = None, pip_install_options: list[str] | None = None, @@ -190,7 +188,7 @@ class TaskDecoratorCollection: multiple_outputs: bool | None = None, # 'python_callable', 'op_args' and 'op_kwargs' since they are filled by # _PythonVirtualenvDecoratedOperator. - serializer: Serializer | None = None, + serializer: Literal["pickle", "cloudpickle", "dill"] | None = None, templates_dict: Mapping[str, Any] | None = None, show_return_value_in_logs: bool = True, env_vars: dict[str, str] | None = None, @@ -255,7 +253,7 @@ class TaskDecoratorCollection: # _PythonVirtualenvDecoratedOperator. requirements: None | Iterable[str] | str = None, python_version: None | str | int | float = None, - serializer: Serializer | None = None, + serializer: Literal["pickle", "cloudpickle", "dill"] | None = None, system_site_packages: bool = True, templates_dict: Mapping[str, Any] | None = None, pip_install_options: list[str] | None = None, @@ -318,7 +316,7 @@ class TaskDecoratorCollection: multiple_outputs: bool | None = None, # 'python_callable', 'op_args' and 'op_kwargs' since they are filled by # _PythonVirtualenvDecoratedOperator. - serializer: Serializer | None = None, + serializer: Literal["pickle", "cloudpickle", "dill"] | None = None, templates_dict: Mapping[str, Any] | None = None, show_return_value_in_logs: bool = True, use_dill: bool = False, @@ -382,7 +380,7 @@ class TaskDecoratorCollection: *, multiple_outputs: bool | None = None, python_command: str = "python3", - serializer: Serializer | None = None, + serializer: Literal["pickle", "cloudpickle", "dill"] | None = None, use_dill: bool = False, # Added by _DockerDecoratedOperator. # 'command', 'retrieve_output', and 'retrieve_output_path' are filled by # _DockerDecoratedOperator. From d02024a606a915facd891d5a221b18b256398523 Mon Sep 17 00:00:00 2001 From: phi Date: Fri, 9 Aug 2024 21:44:26 +0900 Subject: [PATCH 09/10] fix: compat errors --- airflow/providers/docker/decorators/docker.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/airflow/providers/docker/decorators/docker.py b/airflow/providers/docker/decorators/docker.py index be000da48efd9..2d1ae963e380c 100644 --- a/airflow/providers/docker/decorators/docker.py +++ b/airflow/providers/docker/decorators/docker.py @@ -20,11 +20,10 @@ import os import warnings from tempfile import TemporaryDirectory -from typing import TYPE_CHECKING, Callable, Sequence +from typing import TYPE_CHECKING, Any, Callable, Literal, Sequence from airflow.decorators.base import DecoratedOperator, task_decorator_factory from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning -from airflow.operators.python import _SERIALIZERS, _SerializerTypeDef from airflow.providers.docker.operators.docker import DockerOperator from airflow.utils.python_virtualenv import write_python_script @@ -32,6 +31,47 @@ from airflow.decorators.base import TaskDecorator from airflow.utils.context import Context + Serializer = Literal["pickle", "dill", "cloudpickle"] + +try: + from airflow.operators.python import _SERIALIZERS +except ImportError: + import logging + + import lazy_object_proxy + + log = logging.getLogger(__name__) + + def _load_pickle(): + import pickle + + return pickle + + def _load_dill(): + try: + import dill + except ModuleNotFoundError: + log.error("Unable to import `dill` module. Please please make sure that it installed.") + raise + return dill + + def _load_cloudpickle(): + try: + import cloudpickle + except ModuleNotFoundError: + log.error( + "Unable to import `cloudpickle` module. " + "Please install it with: pip install 'apache-airflow[cloudpickle]'" + ) + raise + return cloudpickle + + _SERIALIZERS: dict[Serializer, Any] = { + "pickle": lazy_object_proxy.Proxy(_load_pickle), + "dill": lazy_object_proxy.Proxy(_load_dill), + "cloudpickle": lazy_object_proxy.Proxy(_load_cloudpickle), + } + def _generate_decode_command(env_var, file, python_command): # We don't need `f.close()` as the interpreter is about to exit anyway @@ -83,7 +123,7 @@ def __init__( use_dill=False, python_command="python3", expect_airflow: bool = True, - serializer: _SerializerTypeDef | None = None, + serializer: Serializer | None = None, **kwargs, ) -> None: if use_dill: @@ -110,7 +150,7 @@ def __init__( self.python_command = python_command self.expect_airflow = expect_airflow self.use_dill = serializer == "dill" - self.serializer: _SerializerTypeDef = serializer + self.serializer: Serializer = serializer super().__init__( command=command, retrieve_output=True, retrieve_output_path="/tmp/script.out", **kwargs From c7b54a13873591abcd5b28d0c7c16f4d0c3e7033 Mon Sep 17 00:00:00 2001 From: phi Date: Fri, 9 Aug 2024 21:45:48 +0900 Subject: [PATCH 10/10] fix: static error --- airflow/providers/docker/decorators/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/docker/decorators/docker.py b/airflow/providers/docker/decorators/docker.py index 2d1ae963e380c..9812e5fc57488 100644 --- a/airflow/providers/docker/decorators/docker.py +++ b/airflow/providers/docker/decorators/docker.py @@ -66,7 +66,7 @@ def _load_cloudpickle(): raise return cloudpickle - _SERIALIZERS: dict[Serializer, Any] = { + _SERIALIZERS: dict[Serializer, Any] = { # type: ignore[no-redef] "pickle": lazy_object_proxy.Proxy(_load_pickle), "dill": lazy_object_proxy.Proxy(_load_dill), "cloudpickle": lazy_object_proxy.Proxy(_load_cloudpickle),