From fcbce9026a668de49a1834102882ae817924770d Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:01:45 +0200 Subject: [PATCH 01/31] fix basic incompatibilities with old Python3.5 --- cloudpickle/cloudpickle.py | 7 ++++++- tests/cloudpickle_test.py | 13 +++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 73ca30cc7..30e9cec45 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -61,7 +61,7 @@ import typing from enum import Enum -from typing import Generic, Union, Tuple, Callable, ClassVar +from typing import Generic, Union, Tuple, Callable from pickle import _Pickler as Pickler from pickle import _getattribute from io import BytesIO @@ -73,6 +73,11 @@ except ImportError: _typing_extensions = Literal = Final = None +if sys.version_info > (3, 5, 2): + from typing import ClassVar +else: # pragma: no cover + ClassVar = None + # cloudpickle is meant for inter process communication: we expect all # communicating processes to run the same Python version hence we favor diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 126bb310a..b4051f0f2 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2182,10 +2182,10 @@ def _all_types_to_test(): class C(typing.Generic[T]): pass - return [ + types_to_test = [ C, C[int], - T, typing.Any, typing.NoReturn, typing.Optional, - typing.Generic, typing.Union, typing.ClassVar, + T, typing.Any, typing.Optional, + typing.Generic, typing.Union, typing.Optional[int], typing.Generic[T], typing.Callable[[int], typing.Any], @@ -2193,10 +2193,15 @@ class C(typing.Generic[T]): typing.Callable[[], typing.Any], typing.Tuple[int, ...], typing.Tuple[int, C[int]], - typing.ClassVar[C[int]], typing.List[int], typing.Dict[int, str], ] + if sys.version_info > (3, 5, 2): + types_to_test.append(typing.ClassVar) + types_to_test.append(typing.ClassVar[C[int]]) + if sys.version_info > (3, 5, 3): + types_to_test.append(typing.NoReturn) + return types_to_test if __name__ == '__main__': From 3fa3227391fd74140b8ad52c1924f7e264ccc7be Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:06:31 +0200 Subject: [PATCH 02/31] fix pickling of type hints in Python 3.5.[0-2] --- cloudpickle/cloudpickle.py | 52 ++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 30e9cec45..60e0cccd5 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -432,10 +432,24 @@ def _extract_class_dict(cls): if sys.version_info[:2] < (3, 7): # pragma: no branch def _is_parametrized_type_hint(obj): # This is very cheap but might generate false positives. - origin = getattr(obj, '__origin__', None) # typing Constructs - values = getattr(obj, '__values__', None) # typing_extensions.Literal - type_ = getattr(obj, '__type__', None) # typing_extensions.Final - return origin is not None or values is not None or type_ is not None + # genral typing Constructs + is_typing = getattr(obj, '__origin__', None) is not None + + # typing_extensions.Literal + is_litteral = getattr(obj, '__values__', None) is not None + + # typing_extensions.Final + is_final = getattr(obj, '__type__', None) is not None + + # typing.Union/Tuple for old Python 3.5 + is_union = getattr(obj, '__union_params__', None) is not None + is_tuple = getattr(obj, '__tuple_params__', None) + is_callable = ( + getattr(obj, '__result__', None) is not None and + getattr(obj, '__args__', None) is not None + ) + return any((is_typing, is_litteral, is_final, is_union, is_tuple, + is_callable)) def _create_parametrized_type_hint(origin, args): return origin[args] @@ -971,14 +985,32 @@ def _save_parametrized_type_hint(self, obj): initargs = (Final, obj.__type__) elif type(obj) is type(ClassVar): initargs = (ClassVar, obj.__type__) - elif type(obj) in [type(Union), type(Tuple), type(Generic)]: - initargs = (obj.__origin__, obj.__args__) + elif type(obj) is type(Generic): + if sys.version_info < (3, 5, 2): # pragma: no cover + initargs = (obj.__origin__, obj.__parameters__) + else: + initargs = (obj.__origin__, obj.__args__) + elif type(obj) is type(Union): + if sys.version_info < (3, 5, 2): # pragma: no cover + initargs = (Union, obj.__union_params__) + else: + initargs = (Union, obj.__args__) + elif type(obj) is type(Tuple): + if sys.version_info < (3, 5, 2): # pragma: no cover + initargs = (Tuple, obj.__union_params__) + else: + initargs = (Tuple, obj.__args__) elif type(obj) is type(Callable): - args = obj.__args__ - if args[0] is Ellipsis: - initargs = (obj.__origin__, args) + if sys.version_info < (3, 5, 2): # pragma: no cover + args = obj.__args__ + result = obj.__result__ + else: + (*args, result) = obj.__args__ + + if args is Ellipsis: + initargs = (Callable, (args, result)) else: - initargs = (obj.__origin__, (list(args[:-1]), args[-1])) + initargs = (Callable, (list(args), result)) else: # pragma: no cover raise pickle.PicklingError( "Cloudpickle Error: Unknown type {}".format(type(obj)) From 732343ecbe83c0d8beb07791b948feab25d70949 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:07:04 +0200 Subject: [PATCH 03/31] modify typing test to account for Python 3.5.[0-2] --- tests/cloudpickle_test.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index b4051f0f2..14cda6789 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2083,12 +2083,18 @@ class C(typing.Generic[T]): def check_generic(generic, origin, type_value): assert generic.__origin__ is origin - assert len(generic.__args__) == 1 - assert generic.__args__[0] is type_value - assert len(origin.__orig_bases__) == 1 - ob = origin.__orig_bases__[0] - assert ob.__origin__ is typing.Generic + if sys.version_info > (3, 5, 2): + assert len(generic.__args__) == 1 + assert generic.__args__[0] is type_value + assert len(origin.__orig_bases__) == 1 + ob = origin.__orig_bases__[0] + assert ob.__origin__ is typing.Generic + else: # Python 3.5.[0-1-2] + assert len(generic.__parameters__) == 1 + assert generic.__parameters__[0] is type_value + assert len(origin.__bases__) == 1 + ob = origin.__bases__[0] assert len(ob.__parameters__) == 1 return "ok" From 3adc5be94c7a92c3d513b633eeb3f342b4c186a6 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:34:33 +0200 Subject: [PATCH 04/31] fixup! fix pickling of type hints in Python 3.5.[0-2] --- cloudpickle/cloudpickle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 60e0cccd5..957219e1c 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -997,7 +997,7 @@ def _save_parametrized_type_hint(self, obj): initargs = (Union, obj.__args__) elif type(obj) is type(Tuple): if sys.version_info < (3, 5, 2): # pragma: no cover - initargs = (Tuple, obj.__union_params__) + initargs = (Tuple, obj.__tuple_params__) else: initargs = (Tuple, obj.__args__) elif type(obj) is type(Callable): From d2b8aa660e2b70698a50afa938eb130eb1b07148 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:34:54 +0200 Subject: [PATCH 05/31] fixup! fix pickling of type hints in Python 3.5.[0-2] --- cloudpickle/cloudpickle.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 957219e1c..34a872310 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -1004,13 +1004,18 @@ def _save_parametrized_type_hint(self, obj): if sys.version_info < (3, 5, 2): # pragma: no cover args = obj.__args__ result = obj.__result__ + if args != Ellipsis: + if isinstance(args, tuple): + args = list(args) + else: + args = [args] else: (*args, result) = obj.__args__ - - if args is Ellipsis: - initargs = (Callable, (args, result)) - else: - initargs = (Callable, (list(args), result)) + if len(args) == 1 and args[0] is Ellipsis: + args = Ellipsis + else: + args = list(args) + initargs = (Callable, (args, result)) else: # pragma: no cover raise pickle.PicklingError( "Cloudpickle Error: Unknown type {}".format(type(obj)) From 96c1d2f02870b6ec5c4288a0915b99d8bde44513 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:37:18 +0200 Subject: [PATCH 06/31] update changelog --- CHANGES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c4857e9e7..7c9494063 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,9 @@ 1.4.1 (in development) ====================== - +- Fix incompatibilities between cloudpickle 1.4.0 and Python 3.5.0/1/2 + introduced by the new support of cloudpickle for pickling typing constructs. + ([issue #360](https://github.com/cloudpipe/cloudpickle/issues/360)) 1.4.0 ===== From 7996513980240dc8e642fcedb093b5e6b576ee25 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:38:45 +0200 Subject: [PATCH 07/31] fix mistake in parametrized type hint checker --- cloudpickle/cloudpickle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 34a872310..346237d32 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -443,7 +443,7 @@ def _is_parametrized_type_hint(obj): # typing.Union/Tuple for old Python 3.5 is_union = getattr(obj, '__union_params__', None) is not None - is_tuple = getattr(obj, '__tuple_params__', None) + is_tuple = getattr(obj, '__tuple_params__', None) is not None is_callable = ( getattr(obj, '__result__', None) is not None and getattr(obj, '__args__', None) is not None From f164a29d7d13c14a2c1a68e0c594a4d3082b94b7 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:45:38 +0200 Subject: [PATCH 08/31] try to test cloudpickle agains Python 3.5.0 in CI --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d7cc99f6e..f6137dbde 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -29,7 +29,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python_version: [3.5, 3.6, 3.7, 3.8, "pypy3"] + python_version: ["3.5.0", 3.6, 3.7, 3.8, "pypy3"] exclude: # Do not test all minor versions on all platforms, especially if they # are not the oldest/newest supported versions From a7e521f85debdd078c2c6e83b1f22965ee93b691 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:50:20 +0200 Subject: [PATCH 09/31] try to use conda python in the CI --- .github/workflows/testing.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f6137dbde..5b13b5906 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -50,9 +50,16 @@ jobs: steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python_version }} + if: matrix.python_version != '3.5.0' uses: actions/setup-python@v1 with: python-version: ${{ matrix.python_version }} + - name: Set up Python ${{ matrix.python_version }} + if: matrix.python_version == '3.5.0' + uses: goanpeca/setup-miniconda@v1 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} - name: Install project and dependencies shell: bash run: | From 4af5e8dceeb8910796d6aa7528d21003e790bdab Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:56:34 +0200 Subject: [PATCH 10/31] fixup! try to use conda python in the CI --- .github/workflows/testing.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5b13b5906..7e49352fa 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -50,12 +50,6 @@ jobs: steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python_version }} - if: matrix.python_version != '3.5.0' - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python_version }} - - name: Set up Python ${{ matrix.python_version }} - if: matrix.python_version == '3.5.0' uses: goanpeca/setup-miniconda@v1 with: auto-update-conda: true From 2c9e644aa639fabfcaec3729663317cad13dc8d4 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 22:59:33 +0200 Subject: [PATCH 11/31] debug miniconda --- .github/workflows/testing.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 7e49352fa..e3cb9dbbe 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -54,6 +54,9 @@ jobs: with: auto-update-conda: true python-version: ${{ matrix.python-version }} + - name: Conda info + shell: bash -l {0} + run: conda info - name: Install project and dependencies shell: bash run: | From d5e058665a546ae700e8db7abff035f4ba752c66 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:01:05 +0200 Subject: [PATCH 12/31] fixup! debug miniconda --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e3cb9dbbe..5da822599 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -56,7 +56,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Conda info shell: bash -l {0} - run: conda info + run: which python - name: Install project and dependencies shell: bash run: | From f50f87f08c37179e9b6c81435b16eb8345fc5dcc Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:05:49 +0200 Subject: [PATCH 13/31] fixup! fixup! debug miniconda --- .github/workflows/testing.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5da822599..c84c9e48b 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -54,7 +54,8 @@ jobs: with: auto-update-conda: true python-version: ${{ matrix.python-version }} - - name: Conda info + activate-environment: test + - name: Test conda Python version shell: bash -l {0} run: which python - name: Install project and dependencies From 726d86d3da439bd6f6272ff3648d957dc89c2547 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:08:34 +0200 Subject: [PATCH 14/31] fixup! fixup! debug miniconda --- .github/workflows/testing.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c84c9e48b..b7ec61123 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -54,10 +54,12 @@ jobs: with: auto-update-conda: true python-version: ${{ matrix.python-version }} - activate-environment: test + - name: Activate conda Python version + shell: bash -l {0} + run: conda activate test - name: Test conda Python version shell: bash -l {0} - run: which python + run: python -V - name: Install project and dependencies shell: bash run: | From 0d834ce8f10240e14dd48c0c3a17f12af4971a58 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:13:32 +0200 Subject: [PATCH 15/31] fixup! fixup! fixup! debug miniconda --- .github/workflows/testing.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b7ec61123..28dbed073 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -54,9 +54,12 @@ jobs: with: auto-update-conda: true python-version: ${{ matrix.python-version }} - - name: Activate conda Python version + - name: Create conda virtual env shell: bash -l {0} - run: conda activate test + run: conda create -y -n testenv python=${{ matrix.python_version }} + - name: Activate virtualenv + shell: bash -l {0} + run: conda activate testenv - name: Test conda Python version shell: bash -l {0} run: python -V From 405b99a54bcd19a097f6ea91555b27fe1768e178 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:16:44 +0200 Subject: [PATCH 16/31] a few last tries --- .github/workflows/testing.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 28dbed073..04e9274da 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -54,12 +54,13 @@ jobs: with: auto-update-conda: true python-version: ${{ matrix.python-version }} - - name: Create conda virtual env + auto-activate-base: false + activate-environment: testenv + - name: Run conda info shell: bash -l {0} - run: conda create -y -n testenv python=${{ matrix.python_version }} - - name: Activate virtualenv - shell: bash -l {0} - run: conda activate testenv + run: | + conda info + conda list - name: Test conda Python version shell: bash -l {0} run: python -V From b1eca2559f8f7a212451be88a1b88b2f4e315cf4 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:22:05 +0200 Subject: [PATCH 17/31] fixup! a few last tries --- .github/workflows/testing.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 04e9274da..2c4ac2190 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -56,17 +56,12 @@ jobs: python-version: ${{ matrix.python-version }} auto-activate-base: false activate-environment: testenv - - name: Run conda info - shell: bash -l {0} - run: | - conda info - conda list - - name: Test conda Python version - shell: bash -l {0} - run: python -V - name: Install project and dependencies shell: bash run: | + conda init bash + source ~/.bashrc + conda activate testenv python -m pip install --upgrade pip python -m pip install -e . python -m pip install -r dev-requirements.txt From abd46d77102e361a88c2d12a01fb619a361c3a8a Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:23:59 +0200 Subject: [PATCH 18/31] I think I get it --- .github/workflows/testing.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2c4ac2190..05213ae2f 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -57,30 +57,27 @@ jobs: auto-activate-base: false activate-environment: testenv - name: Install project and dependencies - shell: bash + shell: bash -l {0} run: | - conda init bash - source ~/.bashrc - conda activate testenv python -m pip install --upgrade pip python -m pip install -e . python -m pip install -r dev-requirements.txt python ci/install_coverage_subprocess_pth.py export - name: Install optional typing_extensions in Python 3.6 - shell: bash + shell: bash -l {0} run: python -m pip install typing-extensions if: matrix.python_version == '3.6' - name: Display Python version - shell: bash + shell: bash -l {0} run: python -c "import sys; print(sys.version)" - name: Look for syntax errors/undefined names - shell: bash + shell: bash -l {0} run: | python -m flake8 . --count --verbose --select=E901,E999,F821,F822,F823 \ --show-source --statistics - name: Test with pytest - shell: bash + shell: bash -l {0} run: | COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \ PYTHONPATH='.:tests' python -m pytest -r s From 1fb09a0b426e0817b4dd2cdb26c7486bf139140f Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:27:26 +0200 Subject: [PATCH 19/31] rollback CI changes --- .github/workflows/testing.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 05213ae2f..d7cc99f6e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -29,7 +29,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python_version: ["3.5.0", 3.6, 3.7, 3.8, "pypy3"] + python_version: [3.5, 3.6, 3.7, 3.8, "pypy3"] exclude: # Do not test all minor versions on all platforms, especially if they # are not the oldest/newest supported versions @@ -50,14 +50,11 @@ jobs: steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python_version }} - uses: goanpeca/setup-miniconda@v1 + uses: actions/setup-python@v1 with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} - auto-activate-base: false - activate-environment: testenv + python-version: ${{ matrix.python_version }} - name: Install project and dependencies - shell: bash -l {0} + shell: bash run: | python -m pip install --upgrade pip python -m pip install -e . @@ -65,19 +62,19 @@ jobs: python ci/install_coverage_subprocess_pth.py export - name: Install optional typing_extensions in Python 3.6 - shell: bash -l {0} + shell: bash run: python -m pip install typing-extensions if: matrix.python_version == '3.6' - name: Display Python version - shell: bash -l {0} + shell: bash run: python -c "import sys; print(sys.version)" - name: Look for syntax errors/undefined names - shell: bash -l {0} + shell: bash run: | python -m flake8 . --count --verbose --select=E901,E999,F821,F822,F823 \ --show-source --statistics - name: Test with pytest - shell: bash -l {0} + shell: bash run: | COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \ PYTHONPATH='.:tests' python -m pytest -r s From 736127cd4998be9259d84ccbd300b7377905c889 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:56:20 +0200 Subject: [PATCH 20/31] fix some Python version check --- cloudpickle/cloudpickle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index f1a297bf2..d34c1da5d 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -73,7 +73,7 @@ except ImportError: _typing_extensions = Literal = Final = None -if sys.version_info > (3, 5, 2): +if sys.version_info >= (3, 5, 3): from typing import ClassVar else: # pragma: no cover ClassVar = None @@ -996,17 +996,17 @@ def _save_parametrized_type_hint(self, obj): else: initargs = (obj.__origin__, obj.__args__) elif type(obj) is type(Union): - if sys.version_info < (3, 5, 2): # pragma: no cover + if sys.version_info < (3, 5, 3): # pragma: no cover initargs = (Union, obj.__union_params__) else: initargs = (Union, obj.__args__) elif type(obj) is type(Tuple): - if sys.version_info < (3, 5, 2): # pragma: no cover + if sys.version_info < (3, 5, 3): # pragma: no cover initargs = (Tuple, obj.__tuple_params__) else: initargs = (Tuple, obj.__args__) elif type(obj) is type(Callable): - if sys.version_info < (3, 5, 2): # pragma: no cover + if sys.version_info < (3, 5, 3): # pragma: no cover args = obj.__args__ result = obj.__result__ if args != Ellipsis: From 6e3621dd1adfa5711a75c25f72409dd949ebe6ab Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:56:47 +0200 Subject: [PATCH 21/31] this cannot be done using version checks --- cloudpickle/cloudpickle.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index d34c1da5d..b30170091 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -991,8 +991,9 @@ def _save_parametrized_type_hint(self, obj): elif type(obj) is type(ClassVar): initargs = (ClassVar, obj.__type__) elif type(obj) is type(Generic): - if sys.version_info < (3, 5, 2): # pragma: no cover - initargs = (obj.__origin__, obj.__parameters__) + parameters = obj.__parameters__ + if len(obj.__parameters__) > 0: + initargs = (obj.__origin__, parameters) else: initargs = (obj.__origin__, obj.__args__) elif type(obj) is type(Union): From ec41dee97c219a3751fa5369c98a3e38cf3cbd72 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 Apr 2020 23:58:43 +0200 Subject: [PATCH 22/31] this cannot be done using version checks --- tests/cloudpickle_test.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 14cda6789..599588232 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2081,26 +2081,31 @@ class C(typing.Generic[T]): with subprocess_worker(protocol=self.protocol) as worker: - def check_generic(generic, origin, type_value): + def check_generic(generic, origin, type_value, use_args): assert generic.__origin__ is origin - if sys.version_info > (3, 5, 2): - assert len(generic.__args__) == 1 - assert generic.__args__[0] is type_value + if sys.version_info >= (3, 5, 3): assert len(origin.__orig_bases__) == 1 ob = origin.__orig_bases__[0] assert ob.__origin__ is typing.Generic else: # Python 3.5.[0-1-2] - assert len(generic.__parameters__) == 1 - assert generic.__parameters__[0] is type_value assert len(origin.__bases__) == 1 ob = origin.__bases__[0] + + if use_args: + assert len(generic.__args__) == 1 + assert generic.__args__[0] is type_value + else: + assert len(generic.__parameters__) == 1 + assert generic.__parameters__[0] is type_value assert len(ob.__parameters__) == 1 return "ok" - assert check_generic(C[int], C, int) == "ok" - assert worker.run(check_generic, C[int], C, int) == "ok" + # backward-compat for old Python3.5 versions + use_args = getattr(C[int], '__args__', ()) != () + assert check_generic(C[int], C, int, use_args) == "ok" + assert worker.run(check_generic, C[int], C, int, use_args) == "ok" def test_locally_defined_class_with_type_hints(self): with subprocess_worker(protocol=self.protocol) as worker: @@ -2202,7 +2207,7 @@ class C(typing.Generic[T]): typing.List[int], typing.Dict[int, str], ] - if sys.version_info > (3, 5, 2): + if sys.version_info[:3] >= (3, 5, 3): types_to_test.append(typing.ClassVar) types_to_test.append(typing.ClassVar[C[int]]) if sys.version_info > (3, 5, 3): From ae643fb7f6a29211d5a6c7df75bf60ed19a14cf6 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 29 Apr 2020 00:28:22 +0200 Subject: [PATCH 23/31] fix typing_extension tests for early Python 3.5 --- tests/cloudpickle_test.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 599588232..c7fac8358 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2127,19 +2127,38 @@ def check_annotations(obj, expected_type): assert check_annotations(obj, type_) == "ok" assert worker.run(check_annotations, obj, type_) == "ok" - def test_generic_extensions(self): + def test_generic_extensions_literal(self): typing_extensions = pytest.importorskip('typing_extensions') - objs = [ - typing_extensions.Literal, - typing_extensions.Final, - typing_extensions.Literal['a'], - typing_extensions.Final[int], + def check_literal_equal(obj1, obj2): + assert obj1.__values__ == obj2.__values__ + assert type(obj1) == type(obj2) == typing_extensions._LiteralMeta + literal_objs = [ + typing_extensions.Literal, typing_extensions.Literal['a'] ] + for obj in literal_objs: + depickled_obj = pickle_depickle(obj, protocol=self.protocol) + try: + # __eq__ does not work for Literal objects in early Python 3.5 + assert depickled_obj == obj + except Exception: + check_literal_equal(obj, depickled_obj) + + def test_generic_extensions_final(self): + typing_extensions = pytest.importorskip('typing_extensions') + + def check_final_equal(obj1, obj2): + assert obj1.__type__ == obj2.__type__ + assert type(obj1) == type(obj2) == typing_extensions._FinalMeta + final_objs = [typing_extensions.Final, typing_extensions.Final[int]] - for obj in objs: + for obj in final_objs: depickled_obj = pickle_depickle(obj, protocol=self.protocol) - assert depickled_obj == obj + try: + # __eq__ does not work for Final objects in early Python 3.5 + assert depickled_obj == obj + except Exception: + check_final_equal(obj, depickled_obj) def test_class_annotations(self): class C: From a9dadd6162256cd25fb57d68096582a59880838f Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 29 Apr 2020 00:30:37 +0200 Subject: [PATCH 24/31] revert TypeVar tracking TypeVar objects are not weakreferable in early Python 3.5 --- cloudpickle/cloudpickle.py | 7 ++----- tests/cloudpickle_test.py | 5 ----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index b30170091..00362f8f4 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -1338,20 +1338,17 @@ def _is_dynamic(module): return _find_spec(module.__name__, pkgpath, module) is None -def _make_typevar(name, bound, constraints, covariant, contravariant, - class_tracker_id): - tv = typing.TypeVar( +def _make_typevar(name, bound, constraints, covariant, contravariant): + return typing.TypeVar( name, *constraints, bound=bound, covariant=covariant, contravariant=contravariant ) - return _lookup_class_or_track(class_tracker_id, tv) def _decompose_typevar(obj): return ( obj.__name__, obj.__bound__, obj.__constraints__, obj.__covariant__, obj.__contravariant__, - _get_or_create_tracker_id(obj), ) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index c7fac8358..93ebff464 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2047,11 +2047,6 @@ def test_pickle_dynamic_typevar(self): for attr in attr_list: assert getattr(T, attr) == getattr(depickled_T, attr) - def test_pickle_dynamic_typevar_tracking(self): - T = typing.TypeVar("T") - T2 = subprocess_pickle_echo(T, protocol=self.protocol) - assert T is T2 - def test_pickle_dynamic_typevar_memoization(self): T = typing.TypeVar('T') depickled_T1, depickled_T2 = pickle_depickle((T, T), From acd23e47082a5754050b786c04c567ae59dc4496 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 29 Apr 2020 00:31:33 +0200 Subject: [PATCH 25/31] fix sys version checks --- tests/cloudpickle_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 93ebff464..1bedd1dcf 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2224,7 +2224,7 @@ class C(typing.Generic[T]): if sys.version_info[:3] >= (3, 5, 3): types_to_test.append(typing.ClassVar) types_to_test.append(typing.ClassVar[C[int]]) - if sys.version_info > (3, 5, 3): + if sys.version_info >= (3, 5, 4): types_to_test.append(typing.NoReturn) return types_to_test From e573c373f085abbbf08c95f013b36de1f2603fd2 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 29 Apr 2020 00:41:44 +0200 Subject: [PATCH 26/31] add a few explanations --- cloudpickle/cloudpickle.py | 2 ++ tests/cloudpickle_test.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 00362f8f4..7911f2a06 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -993,6 +993,8 @@ def _save_parametrized_type_hint(self, obj): elif type(obj) is type(Generic): parameters = obj.__parameters__ if len(obj.__parameters__) > 0: + # in early Python 3.5, __parameters__ was sometimes + # preferred to __args__ initargs = (obj.__origin__, parameters) else: initargs = (obj.__origin__, obj.__args__) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 1bedd1dcf..9f51f8d2c 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2097,7 +2097,8 @@ def check_generic(generic, origin, type_value, use_args): return "ok" - # backward-compat for old Python3.5 versions + # backward-compat for old Python3.5 versions that sometimes relies + # on __parameters__ use_args = getattr(C[int], '__args__', ()) != () assert check_generic(C[int], C, int, use_args) == "ok" assert worker.run(check_generic, C[int], C, int, use_args) == "ok" From c9c8b4617e5a66f7eb48bd7cc2cc317bef7d2b9b Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 29 Apr 2020 09:13:30 +0200 Subject: [PATCH 27/31] Update cloudpickle/cloudpickle.py --- cloudpickle/cloudpickle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 7911f2a06..249f77c61 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -437,7 +437,7 @@ def _extract_class_dict(cls): if sys.version_info[:2] < (3, 7): # pragma: no branch def _is_parametrized_type_hint(obj): # This is very cheap but might generate false positives. - # genral typing Constructs + # general typing Constructs is_typing = getattr(obj, '__origin__', None) is not None # typing_extensions.Literal From cf9e42e98b58843082adf1b9c743f788ea5fc2b0 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 29 Apr 2020 09:26:12 +0200 Subject: [PATCH 28/31] Re-enable dynamic TypeVar tracking --- cloudpickle/cloudpickle.py | 7 +++++-- tests/cloudpickle_test.py | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 249f77c61..b8e01282d 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -1340,17 +1340,20 @@ def _is_dynamic(module): return _find_spec(module.__name__, pkgpath, module) is None -def _make_typevar(name, bound, constraints, covariant, contravariant): - return typing.TypeVar( +def _make_typevar(name, bound, constraints, covariant, contravariant, + class_tracker_id): + tv = typing.TypeVar( name, *constraints, bound=bound, covariant=covariant, contravariant=contravariant ) + return _lookup_class_or_track(class_tracker_id, tv) def _decompose_typevar(obj): return ( obj.__name__, obj.__bound__, obj.__constraints__, obj.__covariant__, obj.__contravariant__, + _get_or_create_tracker_id(obj), ) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 9f51f8d2c..593e3f1e4 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2047,6 +2047,11 @@ def test_pickle_dynamic_typevar(self): for attr in attr_list: assert getattr(T, attr) == getattr(depickled_T, attr) + def test_pickle_dynamic_typevar_tracking(self): + T = typing.TypeVar("T") + T2 = subprocess_pickle_echo(T, protocol=self.protocol) + assert T is T2 + def test_pickle_dynamic_typevar_memoization(self): T = typing.TypeVar('T') depickled_T1, depickled_T2 = pickle_depickle((T, T), From 8c030a36e3bb7eba379dadbff6ce571bc620afa0 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 29 Apr 2020 09:35:49 +0200 Subject: [PATCH 29/31] Apply suggestions from code review --- tests/cloudpickle_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 593e3f1e4..6b8508696 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2088,7 +2088,7 @@ def check_generic(generic, origin, type_value, use_args): assert len(origin.__orig_bases__) == 1 ob = origin.__orig_bases__[0] assert ob.__origin__ is typing.Generic - else: # Python 3.5.[0-1-2] + else: # Python 3.5.[0-1-2], pragma: no cover assert len(origin.__bases__) == 1 ob = origin.__bases__[0] @@ -2102,7 +2102,7 @@ def check_generic(generic, origin, type_value, use_args): return "ok" - # backward-compat for old Python3.5 versions that sometimes relies + # backward-compat for old Python 3.5 versions that sometimes relies # on __parameters__ use_args = getattr(C[int], '__args__', ()) != () assert check_generic(C[int], C, int, use_args) == "ok" From 6359bc593a61f28ff5753d9cd7ee9ecf7e08760d Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 29 Apr 2020 09:44:43 +0200 Subject: [PATCH 30/31] be more explicit when testing Final/Literal --- tests/cloudpickle_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 6b8508696..5c7788474 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2139,10 +2139,10 @@ def check_literal_equal(obj1, obj2): ] for obj in literal_objs: depickled_obj = pickle_depickle(obj, protocol=self.protocol) - try: - # __eq__ does not work for Literal objects in early Python 3.5 + if sys.version_info[:3] >= (3, 5, 3): assert depickled_obj == obj - except Exception: + else: + # __eq__ does not work for Literal objects in early Python 3.5 check_literal_equal(obj, depickled_obj) def test_generic_extensions_final(self): @@ -2155,10 +2155,10 @@ def check_final_equal(obj1, obj2): for obj in final_objs: depickled_obj = pickle_depickle(obj, protocol=self.protocol) - try: - # __eq__ does not work for Final objects in early Python 3.5 + if sys.version_info[:3] >= (3, 5, 3): assert depickled_obj == obj - except Exception: + else: + # __eq__ does not work for Final objects in early Python 3.5 check_final_equal(obj, depickled_obj) def test_class_annotations(self): From e58c34b2bd170a46f0d6b6d434a7c65b097b3258 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 29 Apr 2020 10:04:48 +0200 Subject: [PATCH 31/31] Workaround for Python 3.5.3 --- cloudpickle/cloudpickle.py | 13 +++++++++++-- tests/cloudpickle_test.py | 3 +++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index b8e01282d..c639daab1 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -1346,14 +1346,23 @@ def _make_typevar(name, bound, constraints, covariant, contravariant, name, *constraints, bound=bound, covariant=covariant, contravariant=contravariant ) - return _lookup_class_or_track(class_tracker_id, tv) + if class_tracker_id is not None: + return _lookup_class_or_track(class_tracker_id, tv) + else: # pragma: nocover + # Only for Python 3.5.3 compat. + return tv def _decompose_typevar(obj): + try: + class_tracker_id = _get_or_create_tracker_id(obj) + except TypeError: # pragma: nocover + # TypeVar instances are not weakref-able in Python 3.5.3 + class_tracker_id = None return ( obj.__name__, obj.__bound__, obj.__constraints__, obj.__covariant__, obj.__contravariant__, - _get_or_create_tracker_id(obj), + class_tracker_id, ) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 5c7788474..890d4165d 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2047,6 +2047,9 @@ def test_pickle_dynamic_typevar(self): for attr in attr_list: assert getattr(T, attr) == getattr(depickled_T, attr) + @pytest.mark.skipif( + sys.version_info[:3] == (3, 5, 3), + reason="TypeVar instances are not weakref-able in Python 3.5.3") def test_pickle_dynamic_typevar_tracking(self): T = typing.TypeVar("T") T2 = subprocess_pickle_echo(T, protocol=self.protocol)