From ffe4eab5c91c5fef49ab0d9fcce0d5705fc765b3 Mon Sep 17 00:00:00 2001 From: Robbe Sneyders Date: Fri, 11 May 2018 16:32:26 +0200 Subject: [PATCH 1/2] Futurize typehints subpackage --- sdks/python/apache_beam/typehints/__init__.py | 2 ++ .../apache_beam/typehints/decorators.py | 9 ++++++-- .../typehints/native_type_compatibility.py | 4 ++++ .../native_type_compatibility_test.py | 3 +++ sdks/python/apache_beam/typehints/opcodes.py | 9 +++++--- .../typehints/trivial_inference.py | 13 +++++++++++- .../typehints/trivial_inference_test.py | 4 ++++ .../python/apache_beam/typehints/typecheck.py | 21 ++++++++++++------- .../typehints/typed_pipeline_test.py | 3 +++ .../python/apache_beam/typehints/typehints.py | 21 ++++++++++++++----- .../apache_beam/typehints/typehints_test.py | 16 ++++++++++++-- 11 files changed, 84 insertions(+), 21 deletions(-) diff --git a/sdks/python/apache_beam/typehints/__init__.py b/sdks/python/apache_beam/typehints/__init__.py index e89afa1285a7..23d0b40d07f3 100644 --- a/sdks/python/apache_beam/typehints/__init__.py +++ b/sdks/python/apache_beam/typehints/__init__.py @@ -17,6 +17,8 @@ """A package defining the syntax and decorator semantics for type-hints.""" +from __future__ import absolute_import + # pylint: disable=wildcard-import from apache_beam.typehints.typehints import * from apache_beam.typehints.decorators import * diff --git a/sdks/python/apache_beam/typehints/decorators.py b/sdks/python/apache_beam/typehints/decorators.py index 88160c04660b..6604cf120813 100644 --- a/sdks/python/apache_beam/typehints/decorators.py +++ b/sdks/python/apache_beam/typehints/decorators.py @@ -83,8 +83,13 @@ def foo((a, b)): defined, or before importing a module containing type-hinted functions. """ +from __future__ import absolute_import + import inspect import types +from builtins import next +from builtins import object +from builtins import zip from apache_beam.typehints import native_type_compatibility from apache_beam.typehints import typehints @@ -175,7 +180,7 @@ def with_defaults(self, hints): return IOTypeHints(self.input_types or hints.input_types, self.output_types or hints.output_types) - def __nonzero__(self): + def __bool__(self): return bool(self.input_types or self.output_types) def __repr__(self): @@ -404,7 +409,7 @@ def with_output_types(*return_type_hint, **kwargs): from apache_beam.typehints import with_output_types from apache_beam.typehints import Set - class Coordinate: + class Coordinate(object): def __init__(self, x, y): self.x = x self.y = y diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility.py b/sdks/python/apache_beam/typehints/native_type_compatibility.py index 0be931e8fe2b..87fb2c808167 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility.py @@ -17,8 +17,12 @@ """Module to convert Python's native typing types to Beam types.""" +from __future__ import absolute_import + import collections import typing +from builtins import next +from builtins import range from apache_beam.typehints import typehints diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py index 4171507f3459..73416c279acf 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py @@ -17,8 +17,11 @@ """Test for Beam type compatibility library.""" +from __future__ import absolute_import + import typing import unittest +from builtins import object from apache_beam.typehints import native_type_compatibility from apache_beam.typehints import typehints diff --git a/sdks/python/apache_beam/typehints/opcodes.py b/sdks/python/apache_beam/typehints/opcodes.py index 252bcf50e355..a9874cfbda37 100644 --- a/sdks/python/apache_beam/typehints/opcodes.py +++ b/sdks/python/apache_beam/typehints/opcodes.py @@ -33,8 +33,6 @@ import types from functools import reduce -import six - from . import typehints from .trivial_inference import BoundMethod from .trivial_inference import Const @@ -47,6 +45,11 @@ from .typehints import Tuple from .typehints import Union +try: # Python 2 + unicode # pylint: disable=unicode-builtin +except NameError: # Python 3 + unicode = str + def pop_one(state, unused_arg): del state.stack[-1:] @@ -152,7 +155,7 @@ def binary_true_divide(state, unused_arg): def binary_subscr(state, unused_arg): index = state.stack.pop() base = state.stack.pop() - if base in (str, six.text_type): + if base in (str, unicode): out = base elif (isinstance(index, Const) and isinstance(index.value, int) and isinstance(base, typehints.TupleHint.TupleConstraint)): diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index 92770fba10a5..d8181ad7c789 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -28,11 +28,19 @@ import pprint import sys import types +from builtins import object +from builtins import zip from functools import reduce from apache_beam.typehints import Any from apache_beam.typehints import typehints -from six.moves import builtins + +# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports +try: # Python 2 + import __builtin__ as builtins +except ImportError: # Python 3 + import builtins +# pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports class TypeInferenceError(ValueError): @@ -115,6 +123,9 @@ def __init__(self, f, local_vars=None, stack=()): def __eq__(self, other): return isinstance(other, FrameState) and self.__dict__ == other.__dict__ + def __hash__(self): + return hash(tuple(sorted(self.__dict__.items()))) + def copy(self): return FrameState(self.f, self.vars, self.stack) diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index ad0a41c90592..084b1c58075f 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -16,7 +16,11 @@ # """Tests for apache_beam.typehints.trivial_inference.""" + +from __future__ import absolute_import + import unittest +from builtins import object from apache_beam.typehints import trivial_inference from apache_beam.typehints import typehints diff --git a/sdks/python/apache_beam/typehints/typecheck.py b/sdks/python/apache_beam/typehints/typecheck.py index 8ee0269c6d68..d14aa4d51669 100644 --- a/sdks/python/apache_beam/typehints/typecheck.py +++ b/sdks/python/apache_beam/typehints/typecheck.py @@ -20,13 +20,13 @@ For internal use only; no backwards-compatibility guarantees. """ +from __future__ import absolute_import import collections import inspect -import sys import types -import six +from future.utils import raise_with_traceback from apache_beam import pipeline from apache_beam.pvalue import TaggedOutput @@ -41,6 +41,11 @@ from apache_beam.typehints.typehints import SimpleTypeHintError from apache_beam.typehints.typehints import check_constraint +try: # Python 2 + unicode # pylint: disable=unicode-builtin +except NameError: # Python 3 + unicode = str + class AbstractDoFnWrapper(DoFn): """An abstract class to create wrapper around DoFn""" @@ -87,14 +92,14 @@ def wrapper(self, method, args, kwargs): except TypeCheckError as e: error_msg = ('Runtime type violation detected within ParDo(%s): ' '%s' % (self.full_label, e)) - six.raise_from(TypeCheckError(error_msg), sys.exc_info()[2]) + raise_with_traceback(TypeCheckError(error_msg)) else: return self._check_type(result) def _check_type(self, output): if output is None: return output - elif isinstance(output, (dict,) + six.string_types): + elif isinstance(output, (dict, str, unicode)): object_type = type(output).__name__ raise TypeCheckError('Returning a %s from a ParDo or FlatMap is ' 'discouraged. Please use list("%s") if you really ' @@ -176,12 +181,12 @@ def _type_check(self, type_constraint, datum, is_input): try: check_constraint(type_constraint, datum) except CompositeTypeHintError as e: - six.raise_from(TypeCheckError(e.args[0]), sys.exc_info()[2]) + raise_with_traceback(TypeCheckError(e.args[0])) except SimpleTypeHintError: error_msg = ("According to type-hint expected %s should be of type %s. " "Instead, received '%s', an instance of type %s." % (datum_type, type_constraint, datum, type(datum))) - six.raise_from(TypeCheckError(error_msg), sys.exc_info()[2]) + raise_with_traceback(TypeCheckError(error_msg)) class TypeCheckCombineFn(core.CombineFn): @@ -206,7 +211,7 @@ def add_input(self, accumulator, element, *args, **kwargs): except TypeCheckError as e: error_msg = ('Runtime type violation detected within %s: ' '%s' % (self._label, e)) - six.raise_from(TypeCheckError(error_msg), sys.exc_info()[2]) + raise_with_traceback(TypeCheckError(error_msg)) return self._combinefn.add_input(accumulator, element, *args, **kwargs) def merge_accumulators(self, accumulators, *args, **kwargs): @@ -221,7 +226,7 @@ def extract_output(self, accumulator, *args, **kwargs): except TypeCheckError as e: error_msg = ('Runtime type violation detected within %s: ' '%s' % (self._label, e)) - six.raise_from(TypeCheckError(error_msg), sys.exc_info()[2]) + raise_with_traceback(TypeCheckError(error_msg)) return result diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py index 598847e023ce..feee48623033 100644 --- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py +++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py @@ -16,6 +16,9 @@ # """Unit tests for the type-hint objects and decorators.""" + +from __future__ import absolute_import + import inspect import typing import unittest diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py index af9e1fef70b2..44c95bf49808 100644 --- a/sdks/python/apache_beam/typehints/typehints.py +++ b/sdks/python/apache_beam/typehints/typehints.py @@ -63,12 +63,17 @@ """ +from __future__ import absolute_import + import collections import copy import sys import types +from builtins import next +from builtins import object +from builtins import zip -import six +from future.utils import with_metaclass __all__ = [ 'Any', @@ -411,17 +416,23 @@ def __eq__(self, other): def __repr__(self): return 'Any' + def __hash__(self): + return hash(id(self)) + def type_check(self, instance): pass class TypeVariable(AnyTypeConstraint): + def __init__(self, name): + self.name = name + def __eq__(self, other): return type(self) == type(other) and self.name == other.name - def __init__(self, name): - self.name = name + def __hash__(self): + return hash(id(self)) def __repr__(self): return 'TypeVariable[%s]' % self.name @@ -992,8 +1003,8 @@ def __getitem__(self, type_param): IteratorTypeConstraint = IteratorHint.IteratorTypeConstraint -@six.add_metaclass(GetitemConstructor) -class WindowedTypeConstraint(TypeConstraint): +class WindowedTypeConstraint(with_metaclass(GetitemConstructor, + TypeConstraint)): """A type constraint for WindowedValue objects. Mostly for internal use. diff --git a/sdks/python/apache_beam/typehints/typehints_test.py b/sdks/python/apache_beam/typehints/typehints_test.py index 70ebcb3a3ff1..897fbdbbdc99 100644 --- a/sdks/python/apache_beam/typehints/typehints_test.py +++ b/sdks/python/apache_beam/typehints/typehints_test.py @@ -16,9 +16,15 @@ # """Unit tests for the type-hint objects and decorators.""" + +from __future__ import absolute_import + import functools import inspect import unittest +from builtins import next +from builtins import object +from builtins import range import apache_beam.typehints.typehints as typehints from apache_beam.typehints import Any @@ -1034,8 +1040,14 @@ class DecoratorHelpers(TypeHintTestCase): def test_hint_helper(self): self.assertTrue(is_consistent_with(Any, int)) self.assertTrue(is_consistent_with(int, Any)) - self.assertTrue(is_consistent_with(str, object)) - self.assertFalse(is_consistent_with(object, str)) + # object builtin is shadowed by object imported from future.builtins on + # Python 2. Get the native object class from its baseclass. + try: + native_object = object.__bases__[0] + except IndexError: + native_object = object + self.assertTrue(is_consistent_with(str, native_object)) + self.assertFalse(is_consistent_with(native_object, str)) self.assertTrue(is_consistent_with(str, Union[str, int])) self.assertFalse(is_consistent_with(Union[str, int], str)) From 7b79c0d3791906893adcf5bc27595d8abbb2dea0 Mon Sep 17 00:00:00 2001 From: Robbe Sneyders Date: Thu, 21 Jun 2018 22:57:02 +0200 Subject: [PATCH 2/2] Address PR comments --- .../typehints/native_type_compatibility_test.py | 1 - .../apache_beam/typehints/trivial_inference_test.py | 1 - sdks/python/apache_beam/typehints/typecheck.py | 2 +- sdks/python/apache_beam/typehints/typehints.py | 3 ++- sdks/python/apache_beam/typehints/typehints_test.py | 11 ++--------- sdks/python/tox.ini | 1 + 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py index 73416c279acf..2abde69a95f3 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py @@ -21,7 +21,6 @@ import typing import unittest -from builtins import object from apache_beam.typehints import native_type_compatibility from apache_beam.typehints import typehints diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index 084b1c58075f..9ce78fe4e5c7 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -20,7 +20,6 @@ from __future__ import absolute_import import unittest -from builtins import object from apache_beam.typehints import trivial_inference from apache_beam.typehints import typehints diff --git a/sdks/python/apache_beam/typehints/typecheck.py b/sdks/python/apache_beam/typehints/typecheck.py index d14aa4d51669..0754ea9ed479 100644 --- a/sdks/python/apache_beam/typehints/typecheck.py +++ b/sdks/python/apache_beam/typehints/typecheck.py @@ -99,7 +99,7 @@ def wrapper(self, method, args, kwargs): def _check_type(self, output): if output is None: return output - elif isinstance(output, (dict, str, unicode)): + elif isinstance(output, (dict, bytes, str, unicode)): object_type = type(output).__name__ raise TypeCheckError('Returning a %s from a ParDo or FlatMap is ' 'discouraged. Please use list("%s") if you really ' diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py index 44c95bf49808..5da65fa352d4 100644 --- a/sdks/python/apache_beam/typehints/typehints.py +++ b/sdks/python/apache_beam/typehints/typehints.py @@ -70,7 +70,6 @@ import sys import types from builtins import next -from builtins import object from builtins import zip from future.utils import with_metaclass @@ -417,6 +416,7 @@ def __repr__(self): return 'Any' def __hash__(self): + # TODO(BEAM - 3730) return hash(id(self)) def type_check(self, instance): @@ -432,6 +432,7 @@ def __eq__(self, other): return type(self) == type(other) and self.name == other.name def __hash__(self): + # TODO(BEAM - 3730) return hash(id(self)) def __repr__(self): diff --git a/sdks/python/apache_beam/typehints/typehints_test.py b/sdks/python/apache_beam/typehints/typehints_test.py index 897fbdbbdc99..53cdece0ce4d 100644 --- a/sdks/python/apache_beam/typehints/typehints_test.py +++ b/sdks/python/apache_beam/typehints/typehints_test.py @@ -23,7 +23,6 @@ import inspect import unittest from builtins import next -from builtins import object from builtins import range import apache_beam.typehints.typehints as typehints @@ -1040,14 +1039,8 @@ class DecoratorHelpers(TypeHintTestCase): def test_hint_helper(self): self.assertTrue(is_consistent_with(Any, int)) self.assertTrue(is_consistent_with(int, Any)) - # object builtin is shadowed by object imported from future.builtins on - # Python 2. Get the native object class from its baseclass. - try: - native_object = object.__bases__[0] - except IndexError: - native_object = object - self.assertTrue(is_consistent_with(str, native_object)) - self.assertFalse(is_consistent_with(native_object, str)) + self.assertTrue(is_consistent_with(str, object)) + self.assertFalse(is_consistent_with(object, str)) self.assertTrue(is_consistent_with(str, Union[str, int])) self.assertFalse(is_consistent_with(Union[str, int], str)) diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index c80cefd24cb0..b7b4f87c825c 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -113,6 +113,7 @@ modules = apache_beam/pvalue_test apache_beam/testing apache_beam/tools + apache_beam/typehints commands = python --version pip --version