From 74487956582506ef323e705d4bb06ab16ece3b7a Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 7 May 2020 02:19:29 -0700 Subject: [PATCH 01/11] Use `cloudpickle_fast` if protocol 5 is used --- cloudpickle/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudpickle/__init__.py b/cloudpickle/__init__.py index c3ecfdbc7..c48b613ed 100644 --- a/cloudpickle/__init__.py +++ b/cloudpickle/__init__.py @@ -1,11 +1,10 @@ from __future__ import absolute_import -import sys import pickle from cloudpickle.cloudpickle import * -if sys.version_info[:2] >= (3, 8): +if pickle.HIGHEST_PROTOCOL >= 5: from cloudpickle.cloudpickle_fast import CloudPickler, dumps, dump __version__ = '1.5.0dev0' From 86bf5eca8b6282314b3738126ef7fdb0188c3e62 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 10 May 2020 20:38:25 -0700 Subject: [PATCH 02/11] Add `compat` Tries to use `pickle5` for `pickle` if available on older Python versions. --- cloudpickle/compat.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 cloudpickle/compat.py diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py new file mode 100644 index 000000000..005f7c7f7 --- /dev/null +++ b/cloudpickle/compat.py @@ -0,0 +1,13 @@ +import sys + + +if sys.version_info.major == 3 and sys.version_info.minor < 8: + try: + import pickle5 as pickle + import pickle5._pickle as _pickle + except ImportError: + import pickle + import _pickle +else: + import pickle + import _pickle From 72a6bc7fad2883306e533619170b03b7a20b67ae Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 10 May 2020 20:52:10 -0700 Subject: [PATCH 03/11] Use `compat` to provide `pickle` or `pickle5` --- cloudpickle/__init__.py | 2 +- cloudpickle/cloudpickle.py | 8 +++++--- cloudpickle/cloudpickle_fast.py | 7 +++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cloudpickle/__init__.py b/cloudpickle/__init__.py index c48b613ed..e61c01302 100644 --- a/cloudpickle/__init__.py +++ b/cloudpickle/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -import pickle +from cloudpickle.compat import pickle from cloudpickle.cloudpickle import * diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index c639daab1..38b73a171 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -50,7 +50,7 @@ import logging import opcode import operator -import pickle +from cloudpickle.compat import pickle import platform import struct import sys @@ -62,8 +62,6 @@ from enum import Enum from typing import Generic, Union, Tuple, Callable -from pickle import _Pickler as Pickler -from pickle import _getattribute from io import BytesIO from importlib._bootstrap import _find_spec @@ -79,6 +77,10 @@ ClassVar = None +# Aliases using the compat module +Pickler = pickle._Pickler +_getattribute = pickle._getattribute + # cloudpickle is meant for inter process communication: we expect all # communicating processes to run the same Python version hence we favor # communication speed over compatibility: diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 7df95f8dc..0559be1c6 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -15,15 +15,13 @@ import io import itertools import logging -import _pickle -import pickle +from cloudpickle.compat import pickle +from cloudpickle.compat import _pickle import sys import types import weakref import typing -from _pickle import Pickler - from .cloudpickle import ( _is_dynamic, _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, _find_imported_submodules, _get_cell_contents, _is_importable_by_name, _builtin_type, @@ -31,6 +29,7 @@ _extract_class_dict, dynamic_subimport, subimport, _typevar_reduce, _get_bases, ) +Pickler = _pickle.Pickler load, loads = _pickle.load, _pickle.loads From e98d98edfacb44727333e9ff7ca1fffe7465736e Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 10 May 2020 20:54:17 -0700 Subject: [PATCH 04/11] Require `pickle5` for CI testing --- dev-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 8dc4512d4..59c04a1a8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,6 +3,8 @@ flake8 pytest pytest-cov psutil +# To test on older Python versions +pickle5; python_version <= '3.7' # To be able to test tornado coroutines tornado # To be able to test numpy specific things From a8da6e32a4c5bfc4855728be06c99fe588a5783c Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 10 May 2020 20:57:24 -0700 Subject: [PATCH 05/11] Ignore unused imports in `compat` module --- cloudpickle/compat.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index 005f7c7f7..977b8785d 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -3,11 +3,11 @@ if sys.version_info.major == 3 and sys.version_info.minor < 8: try: - import pickle5 as pickle - import pickle5._pickle as _pickle + import pickle5 as pickle # noqa: F401 + import pickle5._pickle as _pickle # noqa: F401 except ImportError: - import pickle - import _pickle + import pickle # noqa: F401 + import _pickle # noqa: F401 else: - import pickle - import _pickle + import pickle # noqa: F401 + import _pickle # noqa: F401 From ad7aa6c26bda9c6e59111693491261e6d50a71f3 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 10 May 2020 21:00:17 -0700 Subject: [PATCH 06/11] Install `pickle5` only on Python 3.6 and 3.7 --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 59c04a1a8..763129165 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,7 +4,7 @@ pytest pytest-cov psutil # To test on older Python versions -pickle5; python_version <= '3.7' +pickle5; python_version >= '3.6' and python_version <= '3.7' # To be able to test tornado coroutines tornado # To be able to test numpy specific things From cd12e10dc579744e5452b87358ae3d21b905cf8e Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 10 May 2020 21:04:14 -0700 Subject: [PATCH 07/11] Import `_getattribute` from `pickle` directly It seems `pickle5` doesn't include this or an alias to it. So just import it from `pickle` directly. --- cloudpickle/cloudpickle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 38b73a171..9605c582e 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -62,6 +62,7 @@ from enum import Enum from typing import Generic, Union, Tuple, Callable +from pickle import _getattribute from io import BytesIO from importlib._bootstrap import _find_spec @@ -79,7 +80,6 @@ # Aliases using the compat module Pickler = pickle._Pickler -_getattribute = pickle._getattribute # cloudpickle is meant for inter process communication: we expect all # communicating processes to run the same Python version hence we favor From 3b5ec5d02d56fe5e03a270a13e107dadf92d9709 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 10 May 2020 21:09:06 -0700 Subject: [PATCH 08/11] Only handle `CellType` when available As `CellType` was added in Python 3.8, it won't be available in Python 3.6 or 3.7. So check before adding it to the dispatch table. --- cloudpickle/cloudpickle_fast.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 0559be1c6..e916612df 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -415,7 +415,8 @@ class CloudPickler(Pickler): dispatch[memoryview] = _memoryview_reduce dispatch[property] = _property_reduce dispatch[staticmethod] = _classmethod_reduce - dispatch[types.CellType] = _cell_reduce + if hasattr(types, "CellType"): + dispatch[types.CellType] = _cell_reduce dispatch[types.CodeType] = _code_reduce dispatch[types.GetSetDescriptorType] = _getset_descriptor_reduce dispatch[types.ModuleType] = _module_reduce From 6734c59e9d983b13521c6d895c527f846b899ce2 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 10 May 2020 21:16:29 -0700 Subject: [PATCH 09/11] Import `pickle` from `compat` in tests --- tests/cloudpickle_file_test.py | 2 +- tests/cloudpickle_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cloudpickle_file_test.py b/tests/cloudpickle_file_test.py index 4f05186e3..6f1099a19 100644 --- a/tests/cloudpickle_file_test.py +++ b/tests/cloudpickle_file_test.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import os -import pickle import shutil import sys import tempfile @@ -10,6 +9,7 @@ import pytest import cloudpickle +from cloudpickle.compat import pickle class CloudPickleFileTests(unittest.TestCase): diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 890d4165d..8f987ffb7 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -9,7 +9,6 @@ import logging import math from operator import itemgetter, attrgetter -import pickle import platform import random import shutil @@ -43,6 +42,7 @@ tornado = None import cloudpickle +from cloudpickle.compat import pickle from cloudpickle.cloudpickle import _is_dynamic from cloudpickle.cloudpickle import _make_empty_cell, cell_set from cloudpickle.cloudpickle import _extract_class_dict, _whichmodule From f27f4bb506bd9c723bd7e2eb6900023934211ced Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 10 May 2020 21:21:44 -0700 Subject: [PATCH 10/11] Workaround absence of `CellType` --- cloudpickle/cloudpickle_fast.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index e916612df..da3cf606d 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -537,9 +537,11 @@ def _function_getnewargs(self, func): # avoid infinite recursion. if func.__closure__ is None: closure = None - else: + elif hasattr(types, "CellType"): closure = tuple( types.CellType() for _ in range(len(code.co_freevars))) + else: + closure = list(map(_get_cell_contents, func.__closure__)) return code, base_globals, None, None, closure From 8c6518bc72505271a223ab22397e234b6e42f5cf Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 10 May 2020 21:30:44 -0700 Subject: [PATCH 11/11] Handle absence of `co_posonlyargcount` --- cloudpickle/cloudpickle_fast.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index da3cf606d..6844df11d 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -189,14 +189,23 @@ def _enum_getstate(obj): def _code_reduce(obj): """codeobject reducer""" - args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) + if hasattr(obj, "co_posonlyargcount"): + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, + obj.co_cellvars + ) + else: + args = ( + obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, + obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, + obj.co_names, obj.co_varnames, obj.co_filename, + obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_freevars, obj.co_cellvars + ) return types.CodeType, args