diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py b/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py index e03240d23e6a..fe232397f552 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py @@ -21,6 +21,7 @@ import logging import os import random +import sys import tempfile import threading import time @@ -217,6 +218,9 @@ def test_gbk_side_input(self): main | beam.Map(lambda a, b: (a, b), beam.pvalue.AsDict(side)), equal_to([(None, {'a': [1]})])) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def test_multimap_side_input(self): with self.create_pipeline() as p: main = p | 'main' >> beam.Create(['a', 'b']) diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py index 75a0102c2c11..8f68f514712b 100644 --- a/sdks/python/apache_beam/transforms/ptransform_test.py +++ b/sdks/python/apache_beam/transforms/ptransform_test.py @@ -23,6 +23,7 @@ import collections import operator +import os import re import sys import unittest @@ -1463,6 +1464,9 @@ def test_pipeline_runtime_checking_violation_with_side_inputs_via_method(self): "Expected an instance of {}, " "instead found 1.0, an instance of {}.".format(int, float)) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def test_combine_properly_pipeline_type_checks_using_decorator(self): @with_output_types(int) @with_input_types(ints=typehints.Iterable[int]) @@ -1623,6 +1627,9 @@ def test_combine_insufficient_type_hint_information(self): 'ParDo(' 'SortJoin/CombinePerKey/') + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def test_mean_globally_pipeline_checking_satisfied(self): d = (self.p | 'C' >> beam.Create(range(5)).with_output_types(int) @@ -1651,6 +1658,9 @@ def test_mean_globally_pipeline_checking_violated(self): self.assertEqual(expected_msg, e.exception.args[0]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def test_mean_globally_runtime_checking_satisfied(self): self.p._options.view_as(TypeOptions).runtime_type_check = True @@ -1759,6 +1769,9 @@ def test_mean_per_key_runtime_checking_violated(self): self.assertStartswith(e.exception.args[0], expected_msg) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def test_count_globally_pipeline_type_checking_satisfied(self): d = (self.p | 'P' >> beam.Create(range(5)).with_output_types(int) @@ -1768,6 +1781,9 @@ def test_count_globally_pipeline_type_checking_satisfied(self): assert_that(d, equal_to([5])) self.p.run() + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def test_count_globally_runtime_type_checking_satisfied(self): self.p._options.view_as(TypeOptions).runtime_type_check = True @@ -2056,6 +2072,9 @@ def test_runtime_type_check_python_type_error(self): e.exception.args[0]) self.assertFalse(isinstance(e, typehints.TypeCheckError)) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def test_pardo_type_inference(self): self.assertEqual(int, beam.Filter(lambda x: False).infer_output_type(int)) @@ -2067,6 +2086,9 @@ def test_gbk_type_inference(self): typehints.Tuple[str, typehints.Iterable[int]], _GroupByKeyOnly().infer_output_type(typehints.KV[str, int])) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def test_pipeline_inference(self): created = self.p | beam.Create(['a', 'b', 'c']) mapped = created | 'pair with 1' >> beam.Map(lambda x: (x, 1)) @@ -2076,6 +2098,9 @@ def test_pipeline_inference(self): self.assertEqual(typehints.KV[str, typehints.Iterable[int]], grouped.element_type) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def test_inferred_bad_kv_type(self): with self.assertRaises(typehints.TypeCheckError) as e: _ = (self.p diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index e4a22fa53d69..1c1af0b4ecee 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -34,20 +34,32 @@ class TrivialInferenceTest(unittest.TestCase): def assertReturnType(self, expected, f, inputs=()): self.assertEquals(expected, trivial_inference.infer_return_type(f, inputs)) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testIdentity(self): self.assertReturnType(int, lambda x: x, [int]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testIndexing(self): self.assertReturnType(int, lambda x: x[0], [typehints.Tuple[int, str]]) self.assertReturnType(str, lambda x: x[1], [typehints.Tuple[int, str]]) self.assertReturnType(str, lambda x: x[1], [typehints.List[str]]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testTuples(self): self.assertReturnType( typehints.Tuple[typehints.Tuple[()], int], lambda x: ((), x), [int]) self.assertReturnType( typehints.Tuple[str, int, float], lambda x: (x, 0, 1.0), [str]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testGetItem(self): def reverse(ab): return ab[-1], ab[0] @@ -62,6 +74,9 @@ def reverse(ab): self.assertReturnType( typehints.List[int], lambda v: v[::-1], [typehints.List[int]]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testUnpack(self): def reverse(a_b): (a, b) = a_b @@ -84,6 +99,9 @@ def reverse(a_b): self.assertReturnType(any_tuple, reverse, [trivial_inference.Const((1, 2, 3))]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testNoneReturn(self): def func(a): if a == 5: @@ -91,6 +109,9 @@ def func(a): return None self.assertReturnType(typehints.Union[int, type(None)], func, [int]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testSimpleList(self): self.assertReturnType( typehints.List[int], @@ -102,6 +123,9 @@ def testSimpleList(self): lambda xs: list(xs), # List is a disallowed builtin [typehints.Tuple[int, ...]]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testListComprehension(self): self.assertReturnType( typehints.List[int], @@ -130,6 +154,9 @@ def testTupleListComprehension(self): lambda L: [(a, a or b, b) for a, b in L], [typehints.Iterable[typehints.Tuple[str, int]]]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testGenerator(self): def foo(x, y): @@ -140,12 +167,18 @@ def foo(x, y): self.assertReturnType( typehints.Iterable[typehints.Union[int, float]], foo, [int, float]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testGeneratorComprehension(self): self.assertReturnType( typehints.Iterable[int], lambda xs: (x for x in xs), [typehints.Tuple[int, ...]]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testBinOp(self): self.assertReturnType(int, lambda a, b: a + b, [int, int]) self.assertReturnType( @@ -154,6 +187,9 @@ def testBinOp(self): typehints.List[typehints.Union[int, str]], lambda a, b: a + b, [typehints.List[int], typehints.List[str]]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testCall(self): f = lambda x, *args: x self.assertReturnType( @@ -162,22 +198,37 @@ def testCall(self): self.assertReturnType( typehints.Tuple[int, typehints.Any], lambda: (1, f(x=1.0))) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testClosure(self): x = 1 y = 1.0 self.assertReturnType(typehints.Tuple[int, float], lambda: (x, y)) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testGlobals(self): self.assertReturnType(int, lambda: global_int) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testBuiltins(self): self.assertReturnType(int, lambda x: len(x), [typehints.Any]) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testGetAttr(self): self.assertReturnType( typehints.Tuple[str, typehints.Any], lambda: (typehints.__doc__, typehints.fake)) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testMethod(self): class A(object): @@ -187,6 +238,9 @@ def m(self, x): self.assertReturnType(int, lambda: A().m(3)) self.assertReturnType(float, lambda: A.m(A(), 3.0)) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testAlwaysReturnsEarly(self): def some_fn(v): @@ -196,6 +250,9 @@ def some_fn(v): self.assertReturnType(int, some_fn) + @unittest.skipIf(sys.version_info >= (3, 6, 0) and + os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1', + 'This test still needs to be fixed on Python 3.6.') def testDict(self): self.assertReturnType( typehints.Dict[typehints.Any, typehints.Any], lambda: {}) diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index 0ac62e980642..f9caf6eea7d7 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -76,20 +76,23 @@ lint.dependsOn lintPy27 toxTask "lintPy27_3", "py27-lint3" lint.dependsOn lintPy27_3 -toxTask "lintPy3", "py3-lint" -lint.dependsOn lintPy3 +toxTask "lintPy35", "py35-lint" +lint.dependsOn lintPy35 toxTask "testPy2Gcp", "py27-gcp" test.dependsOn testPy2Gcp -toxTask "testPy3Gcp", "py3-gcp" -test.dependsOn testPy3Gcp +toxTask "testPy35Gcp", "py35-gcp" +test.dependsOn testPy35Gcp toxTask "testPython2", "py27" test.dependsOn testPython2 -toxTask "testPython3", "py3" -test.dependsOn testPython3 +toxTask "testPython35", "py35" +test.dependsOn testPython35 + +toxTask "testPython36", "py36" +test.dependsOn testPython36 toxTask "testPy2Cython", "py27-cython" test.dependsOn testPy2Cython @@ -98,12 +101,12 @@ test.dependsOn testPy2Cython // project. testPy2Cython.mustRunAfter testPython2, testPy2Gcp -toxTask "testPy3Cython", "py3-cython" -test.dependsOn testPy3Cython -// Ensure that testPy3Cython runs exclusively to other tests. This line is not +toxTask "testPy35Cython", "py35-cython" +test.dependsOn testPy35Cython +// Ensure that testPy35Cython runs exclusively to other tests. This line is not // actually required, since gradle doesn't do parallel execution within a // project. -testPy3Cython.mustRunAfter testPython3, testPy3Gcp +testPy35Cython.mustRunAfter testPython35, testPy35Gcp toxTask "docs", "docs" assemble.dependsOn docs @@ -113,11 +116,12 @@ toxTask "cover", "cover" task preCommit() { dependsOn "docs" dependsOn "testPy2Cython" - dependsOn "testPy3Cython" + dependsOn "testPy35Cython" dependsOn "testPython2" - dependsOn "testPython3" + dependsOn "testPython35" + dependsOn "testPython36" dependsOn "testPy2Gcp" - dependsOn "testPy3Gcp" + dependsOn "testPy35Gcp" dependsOn "lint" dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT" } diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 6dd17935f08f..cd19a6b3b1c5 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -17,7 +17,7 @@ [tox] # new environments will be excluded by default unless explicitly added to envlist. -envlist = py27,py3,py27-{gcp,cython,lint,lint3},py3-{gcp,cython,lint},docs +envlist = py27,py35,py36,py27-{gcp,cython,lint,lint3},py35-{gcp,cython,lint},docs toxworkdir = {toxinidir}/target/.tox [pycodestyle] @@ -53,7 +53,18 @@ commands = python setup.py nosetests {toxinidir}/scripts/run_tox_cleanup.sh -[testenv:py3] +[testenv:py35] +setenv = + RUN_SKIPPED_PY3_TESTS=0 +commands = + python --version + pip --version + {toxinidir}/scripts/run_tox_cleanup.sh + python apache_beam/examples/complete/autocomplete_test.py + python setup.py nosetests + {toxinidir}/scripts/run_tox_cleanup.sh + +[testenv:py36] setenv = RUN_SKIPPED_PY3_TESTS=0 commands = @@ -78,7 +89,7 @@ commands = python setup.py nosetests {toxinidir}/scripts/run_tox_cleanup.sh -[testenv:py3-cython] +[testenv:py35-cython] # cython tests are only expected to work in linux (2.x and 3.x) # If we want to add other platforms in the future, it should be: # `platform = linux2|darwin|...` @@ -104,7 +115,7 @@ commands = python setup.py nosetests {toxinidir}/scripts/run_tox_cleanup.sh -[testenv:py3-gcp] +[testenv:py35-gcp] setenv = RUN_SKIPPED_PY3_TESTS=0 extras = test,gcp @@ -143,7 +154,7 @@ commands = time {toxinidir}/scripts/run_pylint_2to3.sh -[testenv:py3-lint] +[testenv:py35-lint] deps = pycodestyle==2.3.1 pylint==2.1.1