From 52b5eb9c87dbf44dbd7d1e4e7b3e850785e670f4 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 10 Jun 2020 11:05:39 -0700 Subject: [PATCH 1/6] Support pickle5 in "pickle" serialization protocol --- distributed/protocol/pickle.py | 10 +++++++++- distributed/protocol/tests/test_pickle.py | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py index 6e168947d48..fd2343756a4 100644 --- a/distributed/protocol/pickle.py +++ b/distributed/protocol/pickle.py @@ -1,8 +1,16 @@ import logging -import pickle +import sys import cloudpickle +if sys.version_info < (3, 8): + try: + import pickle5 as pickle + except ImportError: + import pickle +else: + import pickle + HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index 9ee496f5e9f..ea2143c5358 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -1,7 +1,6 @@ from functools import partial import gc from operator import add -import pickle import weakref import sys @@ -10,6 +9,14 @@ from distributed.protocol import deserialize, serialize from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads +if sys.version_info < (3, 8): + try: + import pickle5 as pickle + except ImportError: + import pickle +else: + import pickle + def test_pickle_data(): data = [1, b"123", "123", [123], {}, set()] From 442307a140d4f18327bc458d163c5cfc8904da3d Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 10 Jun 2020 11:05:40 -0700 Subject: [PATCH 2/6] Support pickle5 in the scheduler --- distributed/scheduler.py | 10 +++++++++- distributed/tests/test_scheduler.py | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/distributed/scheduler.py b/distributed/scheduler.py index a50407f7966..44828441e72 100644 --- a/distributed/scheduler.py +++ b/distributed/scheduler.py @@ -12,7 +12,7 @@ from numbers import Number import operator import os -import pickle +import sys import random import warnings import weakref @@ -86,6 +86,14 @@ from .stealing import WorkStealing from .variable import VariableExtension +if sys.version_info < (3, 8): + try: + import pickle5 as pickle + except ImportError: + import pickle +else: + import pickle + logger = logging.getLogger(__name__) diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py index a8cdc18bd15..b4753920ff6 100644 --- a/distributed/tests/test_scheduler.py +++ b/distributed/tests/test_scheduler.py @@ -1,7 +1,6 @@ import asyncio import json import logging -import pickle import operator import re import sys @@ -42,6 +41,14 @@ from distributed.utils_test import loop, nodebug # noqa: F401 from dask.compatibility import apply +if sys.version_info < (3, 8): + try: + import pickle5 as pickle + except ImportError: + import pickle +else: + import pickle + alice = "alice:1234" bob = "bob:1234" From 2a317ce591d8ae850f3426a9b9903b7d73463554 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 10 Jun 2020 11:05:41 -0700 Subject: [PATCH 3/6] Install pickle5 on CI --- continuous_integration/travis/install.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh index b1dfbc9ce1c..2e834a0f8a3 100644 --- a/continuous_integration/travis/install.sh +++ b/continuous_integration/travis/install.sh @@ -71,9 +71,11 @@ if [[ $PYTHON == 3.6 ]]; then conda install -c conda-forge -c defaults contextvars fi -# stacktrace is not currently avaiable for Python 3.8. -# Remove the version check block below when it is avaiable. if [[ $PYTHON != 3.8 ]]; then + # Install backport package for pickle protocol 5 support + conda install -c conda-forge -c defaults 'pickle5>=0.0.10' + # stacktrace is not currently avaiable for Python 3.8. + # Remove the version check block below when it is avaiable. # For low-level profiler, install libunwind and stacktrace from conda-forge # For stacktrace we use --no-deps to avoid upgrade of python conda install -c conda-forge -c defaults libunwind From 847b4522f8ac9642ec6f870e03359cd5b5a92752 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 10 Jun 2020 11:05:42 -0700 Subject: [PATCH 4/6] WIP: Install cloudpickle with pickle5 support --- continuous_integration/travis/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh index 2e834a0f8a3..a917dba2e1a 100644 --- a/continuous_integration/travis/install.sh +++ b/continuous_integration/travis/install.sh @@ -74,6 +74,7 @@ fi if [[ $PYTHON != 3.8 ]]; then # Install backport package for pickle protocol 5 support conda install -c conda-forge -c defaults 'pickle5>=0.0.10' + pip install git+https://github.com/jakirkham/cloudpickle.git@opt_use_pickle5_redux # stacktrace is not currently avaiable for Python 3.8. # Remove the version check block below when it is avaiable. # For low-level profiler, install libunwind and stacktrace from conda-forge From f3692433492a64855e0d354e7796539d1d8ddfdf Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 1 Jul 2020 11:06:10 -0700 Subject: [PATCH 5/6] Require `cloudpickle` version `1.5.0+` This is needed for support with `pickle5`. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 95a681d66c2..516ceb62a8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ click >= 6.6 -cloudpickle >= 1.3.0 +cloudpickle >= 1.5.0 contextvars;python_version<'3.7' dask >= 2.9.0 msgpack >= 0.6.0 From 2792f1ed1cd2e08ad10d71488ca35894cc816ad5 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 1 Jul 2020 12:09:39 -0700 Subject: [PATCH 6/6] Just install `pickle5` for legacy support Drop the `cloudpickle` hack as we now have a releae with our needs met, which we included in our requirements. Also ensure we have the latest `pickle5` with fixes for testing. --- continuous_integration/travis/install.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh index a917dba2e1a..be6711f890d 100644 --- a/continuous_integration/travis/install.sh +++ b/continuous_integration/travis/install.sh @@ -73,8 +73,7 @@ fi if [[ $PYTHON != 3.8 ]]; then # Install backport package for pickle protocol 5 support - conda install -c conda-forge -c defaults 'pickle5>=0.0.10' - pip install git+https://github.com/jakirkham/cloudpickle.git@opt_use_pickle5_redux + conda install -c conda-forge -c defaults 'pickle5>=0.0.11' # stacktrace is not currently avaiable for Python 3.8. # Remove the version check block below when it is avaiable. # For low-level profiler, install libunwind and stacktrace from conda-forge