From 09bc1668be76fc54ac009169de9565998e6c0491 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Thu, 24 Jun 2021 13:18:43 -0500 Subject: [PATCH 1/3] Remove more internal references to deprecated utilities --- distributed/dashboard/components/__init__.py | 4 ++-- distributed/dashboard/components/scheduler.py | 4 ++-- distributed/dashboard/components/shared.py | 4 ++-- distributed/dashboard/components/worker.py | 4 ++-- distributed/diagnostics/task_stream.py | 3 ++- distributed/http/scheduler/info.py | 4 ++-- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py index 7c891d96119..f0798417cd9 100644 --- a/distributed/dashboard/components/__init__.py +++ b/distributed/dashboard/components/__init__.py @@ -30,7 +30,7 @@ from distributed import profile from distributed.dashboard.utils import BOKEH_VERSION, without_property_validation -from distributed.utils import log_errors, parse_timedelta +from distributed.utils import log_errors if dask.config.get("distributed.dashboard.export-tool"): from distributed.dashboard.export_tool import ExportTool @@ -39,7 +39,7 @@ profile_interval = dask.config.get("distributed.worker.profile.interval") -profile_interval = parse_timedelta(profile_interval, default="ms") +profile_interval = dask.utils.parse_timedelta(profile_interval, default="ms") class DashboardComponent: diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py index 7f197aa54d2..8a0bda4b3ef 100644 --- a/distributed/dashboard/components/scheduler.py +++ b/distributed/dashboard/components/scheduler.py @@ -43,7 +43,7 @@ import dask from dask import config -from dask.utils import format_bytes, key_split +from dask.utils import format_bytes, format_time, key_split, parse_timedelta try: import numpy as np @@ -70,7 +70,7 @@ from distributed.diagnostics.task_stream import color_of as ts_color_of from distributed.diagnostics.task_stream import colors as ts_color_lookup from distributed.metrics import time -from distributed.utils import Log, format_time, log_errors, parse_timedelta +from distributed.utils import Log, log_errors if dask.config.get("distributed.dashboard.export-tool"): from distributed.dashboard.export_tool import ExportTool diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py index f50145f0dbb..360f4433aa7 100644 --- a/distributed/dashboard/components/shared.py +++ b/distributed/dashboard/components/shared.py @@ -28,7 +28,7 @@ update, without_property_validation, ) -from distributed.utils import log_errors, parse_timedelta +from distributed.utils import log_errors if dask.config.get("distributed.dashboard.export-tool"): from distributed.dashboard.export_tool import ExportTool @@ -37,7 +37,7 @@ profile_interval = dask.config.get("distributed.worker.profile.interval") -profile_interval = parse_timedelta(profile_interval, default="ms") +profile_interval = dask.utils.parse_timedelta(profile_interval, default="ms") class Processing(DashboardComponent): diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py index 9266e1f2766..e664cd4cae5 100644 --- a/distributed/dashboard/components/worker.py +++ b/distributed/dashboard/components/worker.py @@ -20,7 +20,7 @@ from bokeh.themes import Theme from tlz import merge, partition_all -from dask.utils import format_bytes +from dask.utils import format_bytes, format_time from distributed.dashboard.components import add_periodic_callback from distributed.dashboard.components.shared import ( @@ -32,7 +32,7 @@ from distributed.dashboard.utils import transpose, update, without_property_validation from distributed.diagnostics.progress_stream import color_of from distributed.metrics import time -from distributed.utils import format_time, key_split, log_errors +from distributed.utils import key_split, log_errors logger = logging.getLogger(__name__) diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py index e276c477776..fab787ae07c 100644 --- a/distributed/diagnostics/task_stream.py +++ b/distributed/diagnostics/task_stream.py @@ -2,9 +2,10 @@ from collections import deque import dask +from dask.utils import format_time, parse_timedelta from ..metrics import time -from ..utils import format_time, key_split, parse_timedelta +from ..utils import key_split from .plugin import SchedulerPlugin from .progress_stream import color_of diff --git a/distributed/http/scheduler/info.py b/distributed/http/scheduler/info.py index 588fbc1505f..44197141e9e 100644 --- a/distributed/http/scheduler/info.py +++ b/distributed/http/scheduler/info.py @@ -8,11 +8,11 @@ from tornado import escape from tornado.websocket import WebSocketHandler -from dask.utils import format_bytes +from dask.utils import format_bytes, format_time from ...diagnostics.websocket import WebsocketPlugin from ...metrics import time -from ...utils import format_time, log_errors +from ...utils import log_errors from ..utils import RequestHandler, redirect ns = { From 1e4c9a1cd5fc82d9186ad139d13d68a2abd3e54c Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Thu, 24 Jun 2021 13:24:16 -0500 Subject: [PATCH 2/3] Temporarily remove utilities to ensure all reference are accounted for --- distributed/tests/test_utils.py | 57 +++++++++++++++++---------------- distributed/utils.py | 49 ++++++++++++++-------------- 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py index 6f94ca3d506..a2952f81c86 100644 --- a/distributed/tests/test_utils.py +++ b/distributed/tests/test_utils.py @@ -555,43 +555,44 @@ async def test_offload(): assert (await offload(lambda x, y: x + y, 1, y=2)) == 3 -def test_serialize_for_cli_deprecated(): - with pytest.warns(FutureWarning, match="serialize_for_cli is deprecated"): - from distributed.utils import serialize_for_cli - assert serialize_for_cli is dask.config.serialize +# TODO: Uncomment the tests below before merging +# def test_serialize_for_cli_deprecated(): +# with pytest.warns(FutureWarning, match="serialize_for_cli is deprecated"): +# from distributed.utils import serialize_for_cli +# assert serialize_for_cli is dask.config.serialize -def test_deserialize_for_cli_deprecated(): - with pytest.warns(FutureWarning, match="deserialize_for_cli is deprecated"): - from distributed.utils import deserialize_for_cli - assert deserialize_for_cli is dask.config.deserialize +# def test_deserialize_for_cli_deprecated(): +# with pytest.warns(FutureWarning, match="deserialize_for_cli is deprecated"): +# from distributed.utils import deserialize_for_cli +# assert deserialize_for_cli is dask.config.deserialize -def test_parse_bytes_deprecated(): - with pytest.warns(FutureWarning, match="parse_bytes is deprecated"): - from distributed.utils import parse_bytes - assert parse_bytes is dask.utils.parse_bytes +# def test_parse_bytes_deprecated(): +# with pytest.warns(FutureWarning, match="parse_bytes is deprecated"): +# from distributed.utils import parse_bytes +# assert parse_bytes is dask.utils.parse_bytes -def test_format_bytes_deprecated(): - with pytest.warns(FutureWarning, match="format_bytes is deprecated"): - from distributed.utils import format_bytes - assert format_bytes is dask.utils.format_bytes +# def test_format_bytes_deprecated(): +# with pytest.warns(FutureWarning, match="format_bytes is deprecated"): +# from distributed.utils import format_bytes +# assert format_bytes is dask.utils.format_bytes -def test_format_time_deprecated(): - with pytest.warns(FutureWarning, match="format_time is deprecated"): - from distributed.utils import format_time - assert format_time is dask.utils.format_time +# def test_format_time_deprecated(): +# with pytest.warns(FutureWarning, match="format_time is deprecated"): +# from distributed.utils import format_time +# assert format_time is dask.utils.format_time -def test_funcname_deprecated(): - with pytest.warns(FutureWarning, match="funcname is deprecated"): - from distributed.utils import funcname - assert funcname is dask.utils.funcname +# def test_funcname_deprecated(): +# with pytest.warns(FutureWarning, match="funcname is deprecated"): +# from distributed.utils import funcname +# assert funcname is dask.utils.funcname -def test_parse_timedelta_deprecated(): - with pytest.warns(FutureWarning, match="parse_timedelta is deprecated"): - from distributed.utils import parse_timedelta - assert parse_timedelta is dask.utils.parse_timedelta +# def test_parse_timedelta_deprecated(): +# with pytest.warns(FutureWarning, match="parse_timedelta is deprecated"): +# from distributed.utils import parse_timedelta +# assert parse_timedelta is dask.utils.parse_timedelta diff --git a/distributed/utils.py b/distributed/utils.py index dcea26e628a..4b4e9552189 100644 --- a/distributed/utils.py +++ b/distributed/utils.py @@ -1445,27 +1445,28 @@ def clean_dashboard_address(addrs: AnyType, default_listen_ip: str = "") -> List return addresses -_deprecations = { - "deserialize_for_cli": "dask.config.deserialize", - "serialize_for_cli": "dask.config.serialize", - "format_bytes": "dask.utils.format_bytes", - "format_time": "dask.utils.format_time", - "funcname": "dask.utils.funcname", - "parse_bytes": "dask.utils.parse_bytes", - "parse_timedelta": "dask.utils.parse_timedelta", -} - - -def __getattr__(name): - if name in _deprecations: - use_instead = _deprecations[name] - - warnings.warn( - f"{name} is deprecated and will be removed in a future release. " - f"Please use {use_instead} instead.", - category=FutureWarning, - stacklevel=2, - ) - return import_term(use_instead) - else: - raise AttributeError(f"module {__name__} has no attribute {name}") +# TODO: Uncomment the code below before merging +# _deprecations = { +# "deserialize_for_cli": "dask.config.deserialize", +# "serialize_for_cli": "dask.config.serialize", +# "format_bytes": "dask.utils.format_bytes", +# "format_time": "dask.utils.format_time", +# "funcname": "dask.utils.funcname", +# "parse_bytes": "dask.utils.parse_bytes", +# "parse_timedelta": "dask.utils.parse_timedelta", +# } + + +# def __getattr__(name): +# if name in _deprecations: +# use_instead = _deprecations[name] + +# warnings.warn( +# f"{name} is deprecated and will be removed in a future release. " +# f"Please use {use_instead} instead.", +# category=FutureWarning, +# stacklevel=2, +# ) +# return import_term(use_instead) +# else: +# raise AttributeError(f"module {__name__} has no attribute {name}") From 67f5362ebcf3183a6ef743ae36b8e8a77207eae3 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Thu, 24 Jun 2021 15:59:17 -0500 Subject: [PATCH 3/3] Add deprecations back --- distributed/tests/test_utils.py | 57 ++++++++++++++++----------------- distributed/utils.py | 49 ++++++++++++++-------------- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py index a2952f81c86..6f94ca3d506 100644 --- a/distributed/tests/test_utils.py +++ b/distributed/tests/test_utils.py @@ -555,44 +555,43 @@ async def test_offload(): assert (await offload(lambda x, y: x + y, 1, y=2)) == 3 -# TODO: Uncomment the tests below before merging -# def test_serialize_for_cli_deprecated(): -# with pytest.warns(FutureWarning, match="serialize_for_cli is deprecated"): -# from distributed.utils import serialize_for_cli -# assert serialize_for_cli is dask.config.serialize +def test_serialize_for_cli_deprecated(): + with pytest.warns(FutureWarning, match="serialize_for_cli is deprecated"): + from distributed.utils import serialize_for_cli + assert serialize_for_cli is dask.config.serialize -# def test_deserialize_for_cli_deprecated(): -# with pytest.warns(FutureWarning, match="deserialize_for_cli is deprecated"): -# from distributed.utils import deserialize_for_cli -# assert deserialize_for_cli is dask.config.deserialize +def test_deserialize_for_cli_deprecated(): + with pytest.warns(FutureWarning, match="deserialize_for_cli is deprecated"): + from distributed.utils import deserialize_for_cli + assert deserialize_for_cli is dask.config.deserialize -# def test_parse_bytes_deprecated(): -# with pytest.warns(FutureWarning, match="parse_bytes is deprecated"): -# from distributed.utils import parse_bytes -# assert parse_bytes is dask.utils.parse_bytes +def test_parse_bytes_deprecated(): + with pytest.warns(FutureWarning, match="parse_bytes is deprecated"): + from distributed.utils import parse_bytes + assert parse_bytes is dask.utils.parse_bytes -# def test_format_bytes_deprecated(): -# with pytest.warns(FutureWarning, match="format_bytes is deprecated"): -# from distributed.utils import format_bytes -# assert format_bytes is dask.utils.format_bytes +def test_format_bytes_deprecated(): + with pytest.warns(FutureWarning, match="format_bytes is deprecated"): + from distributed.utils import format_bytes + assert format_bytes is dask.utils.format_bytes -# def test_format_time_deprecated(): -# with pytest.warns(FutureWarning, match="format_time is deprecated"): -# from distributed.utils import format_time -# assert format_time is dask.utils.format_time +def test_format_time_deprecated(): + with pytest.warns(FutureWarning, match="format_time is deprecated"): + from distributed.utils import format_time + assert format_time is dask.utils.format_time -# def test_funcname_deprecated(): -# with pytest.warns(FutureWarning, match="funcname is deprecated"): -# from distributed.utils import funcname -# assert funcname is dask.utils.funcname +def test_funcname_deprecated(): + with pytest.warns(FutureWarning, match="funcname is deprecated"): + from distributed.utils import funcname + assert funcname is dask.utils.funcname -# def test_parse_timedelta_deprecated(): -# with pytest.warns(FutureWarning, match="parse_timedelta is deprecated"): -# from distributed.utils import parse_timedelta -# assert parse_timedelta is dask.utils.parse_timedelta +def test_parse_timedelta_deprecated(): + with pytest.warns(FutureWarning, match="parse_timedelta is deprecated"): + from distributed.utils import parse_timedelta + assert parse_timedelta is dask.utils.parse_timedelta diff --git a/distributed/utils.py b/distributed/utils.py index 4b4e9552189..dcea26e628a 100644 --- a/distributed/utils.py +++ b/distributed/utils.py @@ -1445,28 +1445,27 @@ def clean_dashboard_address(addrs: AnyType, default_listen_ip: str = "") -> List return addresses -# TODO: Uncomment the code below before merging -# _deprecations = { -# "deserialize_for_cli": "dask.config.deserialize", -# "serialize_for_cli": "dask.config.serialize", -# "format_bytes": "dask.utils.format_bytes", -# "format_time": "dask.utils.format_time", -# "funcname": "dask.utils.funcname", -# "parse_bytes": "dask.utils.parse_bytes", -# "parse_timedelta": "dask.utils.parse_timedelta", -# } - - -# def __getattr__(name): -# if name in _deprecations: -# use_instead = _deprecations[name] - -# warnings.warn( -# f"{name} is deprecated and will be removed in a future release. " -# f"Please use {use_instead} instead.", -# category=FutureWarning, -# stacklevel=2, -# ) -# return import_term(use_instead) -# else: -# raise AttributeError(f"module {__name__} has no attribute {name}") +_deprecations = { + "deserialize_for_cli": "dask.config.deserialize", + "serialize_for_cli": "dask.config.serialize", + "format_bytes": "dask.utils.format_bytes", + "format_time": "dask.utils.format_time", + "funcname": "dask.utils.funcname", + "parse_bytes": "dask.utils.parse_bytes", + "parse_timedelta": "dask.utils.parse_timedelta", +} + + +def __getattr__(name): + if name in _deprecations: + use_instead = _deprecations[name] + + warnings.warn( + f"{name} is deprecated and will be removed in a future release. " + f"Please use {use_instead} instead.", + category=FutureWarning, + stacklevel=2, + ) + return import_term(use_instead) + else: + raise AttributeError(f"module {__name__} has no attribute {name}")