Skip to content

[BEAM-5663] Add Python 3.6 tox environment #7988

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aaltay merged 2 commits into from
Mar 13, 2019
Merged

[BEAM-5663] Add Python 3.6 tox environment #7988

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8ada9f1
Add Python 3.6 tox environment
RobbeSneyders Mar 5, 2019
ee41096
Skip failing Python 3.6 tests
RobbeSneyders Mar 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions sdks/python/apache_beam/runners/portability/fn_api_runner_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import logging
import os
import random
import sys
import tempfile
import threading
import time
Expand Down Expand Up @@ -217,6 +218,9 @@ def test_gbk_side_input(self):
main | beam.Map(lambda a, b: (a, b), beam.pvalue.AsDict(side)),
equal_to([(None, {'a': [1]})]))

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
Copy link
Contributor

@tvalentyn tvalentyn Mar 12, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we have Jira's for this and other failures? Could we reference it here?
Thanks.

os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def test_multimap_side_input(self):
with self.create_pipeline() as p:
main = p | 'main' >> beam.Create(['a', 'b'])
Expand Down
25 changes: 25 additions & 0 deletions sdks/python/apache_beam/transforms/ptransform_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

import collections
import operator
import os
import re
import sys
import unittest
Expand Down Expand Up @@ -1463,6 +1464,9 @@ def test_pipeline_runtime_checking_violation_with_side_inputs_via_method(self):
"Expected an instance of {}, "
"instead found 1.0, an instance of {}.".format(int, float))

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def test_combine_properly_pipeline_type_checks_using_decorator(self):
@with_output_types(int)
@with_input_types(ints=typehints.Iterable[int])
Expand Down Expand Up @@ -1623,6 +1627,9 @@ def test_combine_insufficient_type_hint_information(self):
'ParDo('
'SortJoin/CombinePerKey/')

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def test_mean_globally_pipeline_checking_satisfied(self):
d = (self.p
| 'C' >> beam.Create(range(5)).with_output_types(int)
Expand Down Expand Up @@ -1651,6 +1658,9 @@ def test_mean_globally_pipeline_checking_violated(self):

self.assertEqual(expected_msg, e.exception.args[0])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def test_mean_globally_runtime_checking_satisfied(self):
self.p._options.view_as(TypeOptions).runtime_type_check = True

Expand Down Expand Up @@ -1759,6 +1769,9 @@ def test_mean_per_key_runtime_checking_violated(self):

self.assertStartswith(e.exception.args[0], expected_msg)

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def test_count_globally_pipeline_type_checking_satisfied(self):
d = (self.p
| 'P' >> beam.Create(range(5)).with_output_types(int)
Expand All @@ -1768,6 +1781,9 @@ def test_count_globally_pipeline_type_checking_satisfied(self):
assert_that(d, equal_to([5]))
self.p.run()

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def test_count_globally_runtime_type_checking_satisfied(self):
self.p._options.view_as(TypeOptions).runtime_type_check = True

Expand Down Expand Up @@ -2056,6 +2072,9 @@ def test_runtime_type_check_python_type_error(self):
e.exception.args[0])
self.assertFalse(isinstance(e, typehints.TypeCheckError))

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def test_pardo_type_inference(self):
self.assertEqual(int,
beam.Filter(lambda x: False).infer_output_type(int))
Expand All @@ -2067,6 +2086,9 @@ def test_gbk_type_inference(self):
typehints.Tuple[str, typehints.Iterable[int]],
_GroupByKeyOnly().infer_output_type(typehints.KV[str, int]))

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def test_pipeline_inference(self):
created = self.p | beam.Create(['a', 'b', 'c'])
mapped = created | 'pair with 1' >> beam.Map(lambda x: (x, 1))
Expand All @@ -2076,6 +2098,9 @@ def test_pipeline_inference(self):
self.assertEqual(typehints.KV[str, typehints.Iterable[int]],
grouped.element_type)

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def test_inferred_bad_kv_type(self):
with self.assertRaises(typehints.TypeCheckError) as e:
_ = (self.p
Expand Down
57 changes: 57 additions & 0 deletions sdks/python/apache_beam/typehints/trivial_inference_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,20 +34,32 @@ class TrivialInferenceTest(unittest.TestCase):
def assertReturnType(self, expected, f, inputs=()):
self.assertEquals(expected, trivial_inference.infer_return_type(f, inputs))

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
Copy link
Contributor

@tvalentyn tvalentyn Mar 12, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It may be easier to move the skip declaration to skip entire class TrivialInferenceTest.

os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testIdentity(self):
self.assertReturnType(int, lambda x: x, [int])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testIndexing(self):
self.assertReturnType(int, lambda x: x[0], [typehints.Tuple[int, str]])
self.assertReturnType(str, lambda x: x[1], [typehints.Tuple[int, str]])
self.assertReturnType(str, lambda x: x[1], [typehints.List[str]])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testTuples(self):
self.assertReturnType(
typehints.Tuple[typehints.Tuple[()], int], lambda x: ((), x), [int])
self.assertReturnType(
typehints.Tuple[str, int, float], lambda x: (x, 0, 1.0), [str])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testGetItem(self):
def reverse(ab):
return ab[-1], ab[0]
Expand All @@ -62,6 +74,9 @@ def reverse(ab):
self.assertReturnType(
typehints.List[int], lambda v: v[::-1], [typehints.List[int]])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testUnpack(self):
def reverse(a_b):
(a, b) = a_b
Expand All @@ -84,13 +99,19 @@ def reverse(a_b):
self.assertReturnType(any_tuple,
reverse, [trivial_inference.Const((1, 2, 3))])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testNoneReturn(self):
def func(a):
if a == 5:
return a
return None
self.assertReturnType(typehints.Union[int, type(None)], func, [int])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testSimpleList(self):
self.assertReturnType(
typehints.List[int],
Expand All @@ -102,6 +123,9 @@ def testSimpleList(self):
lambda xs: list(xs), # List is a disallowed builtin
[typehints.Tuple[int, ...]])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testListComprehension(self):
self.assertReturnType(
typehints.List[int],
Expand Down Expand Up @@ -130,6 +154,9 @@ def testTupleListComprehension(self):
lambda L: [(a, a or b, b) for a, b in L],
[typehints.Iterable[typehints.Tuple[str, int]]])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testGenerator(self):

def foo(x, y):
Expand All @@ -140,12 +167,18 @@ def foo(x, y):
self.assertReturnType(
typehints.Iterable[typehints.Union[int, float]], foo, [int, float])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testGeneratorComprehension(self):
self.assertReturnType(
typehints.Iterable[int],
lambda xs: (x for x in xs),
[typehints.Tuple[int, ...]])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testBinOp(self):
self.assertReturnType(int, lambda a, b: a + b, [int, int])
self.assertReturnType(
Expand All @@ -154,6 +187,9 @@ def testBinOp(self):
typehints.List[typehints.Union[int, str]], lambda a, b: a + b,
[typehints.List[int], typehints.List[str]])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testCall(self):
f = lambda x, *args: x
self.assertReturnType(
Expand All @@ -162,22 +198,37 @@ def testCall(self):
self.assertReturnType(
typehints.Tuple[int, typehints.Any], lambda: (1, f(x=1.0)))

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testClosure(self):
x = 1
y = 1.0
self.assertReturnType(typehints.Tuple[int, float], lambda: (x, y))

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testGlobals(self):
self.assertReturnType(int, lambda: global_int)

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testBuiltins(self):
self.assertReturnType(int, lambda x: len(x), [typehints.Any])

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testGetAttr(self):
self.assertReturnType(
typehints.Tuple[str, typehints.Any],
lambda: (typehints.__doc__, typehints.fake))

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testMethod(self):

class A(object):
Expand All @@ -187,6 +238,9 @@ def m(self, x):
self.assertReturnType(int, lambda: A().m(3))
self.assertReturnType(float, lambda: A.m(A(), 3.0))

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testAlwaysReturnsEarly(self):

def some_fn(v):
Expand All @@ -196,6 +250,9 @@ def some_fn(v):

self.assertReturnType(int, some_fn)

@unittest.skipIf(sys.version_info >= (3, 6, 0) and
os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
'This test still needs to be fixed on Python 3.6.')
def testDict(self):
self.assertReturnType(
typehints.Dict[typehints.Any, typehints.Any], lambda: {})
Expand Down
30 changes: 17 additions & 13 deletions sdks/python/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,20 +76,23 @@ lint.dependsOn lintPy27
toxTask "lintPy27_3", "py27-lint3"
lint.dependsOn lintPy27_3

toxTask "lintPy3", "py3-lint"
lint.dependsOn lintPy3
toxTask "lintPy35", "py35-lint"
lint.dependsOn lintPy35

toxTask "testPy2Gcp", "py27-gcp"
test.dependsOn testPy2Gcp

toxTask "testPy3Gcp", "py3-gcp"
test.dependsOn testPy3Gcp
toxTask "testPy35Gcp", "py35-gcp"
test.dependsOn testPy35Gcp

toxTask "testPython2", "py27"
test.dependsOn testPython2

toxTask "testPython3", "py3"
test.dependsOn testPython3
toxTask "testPython35", "py35"
test.dependsOn testPython35

toxTask "testPython36", "py36"
test.dependsOn testPython36

toxTask "testPy2Cython", "py27-cython"
test.dependsOn testPy2Cython
Expand All @@ -98,12 +101,12 @@ test.dependsOn testPy2Cython
// project.
testPy2Cython.mustRunAfter testPython2, testPy2Gcp

toxTask "testPy3Cython", "py3-cython"
test.dependsOn testPy3Cython
// Ensure that testPy3Cython runs exclusively to other tests. This line is not
toxTask "testPy35Cython", "py35-cython"
test.dependsOn testPy35Cython
// Ensure that testPy35Cython runs exclusively to other tests. This line is not
// actually required, since gradle doesn't do parallel execution within a
// project.
testPy3Cython.mustRunAfter testPython3, testPy3Gcp
testPy35Cython.mustRunAfter testPython35, testPy35Gcp

toxTask "docs", "docs"
assemble.dependsOn docs
Expand All @@ -113,11 +116,12 @@ toxTask "cover", "cover"
task preCommit() {
dependsOn "docs"
dependsOn "testPy2Cython"
dependsOn "testPy3Cython"
dependsOn "testPy35Cython"
dependsOn "testPython2"
dependsOn "testPython3"
dependsOn "testPython35"
dependsOn "testPython36"
dependsOn "testPy2Gcp"
dependsOn "testPy3Gcp"
dependsOn "testPy35Gcp"
dependsOn "lint"
dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT"
}
Expand Down
21 changes: 16 additions & 5 deletions sdks/python/tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

[tox]
# new environments will be excluded by default unless explicitly added to envlist.
envlist = py27,py3,py27-{gcp,cython,lint,lint3},py3-{gcp,cython,lint},docs
envlist = py27,py35,py36,py27-{gcp,cython,lint,lint3},py35-{gcp,cython,lint},docs
toxworkdir = {toxinidir}/target/.tox

[pycodestyle]
Expand Down Expand Up @@ -53,7 +53,18 @@ commands =
python setup.py nosetests
{toxinidir}/scripts/run_tox_cleanup.sh

[testenv:py3]
[testenv:py35]
setenv =
RUN_SKIPPED_PY3_TESTS=0
commands =
python --version
pip --version
{toxinidir}/scripts/run_tox_cleanup.sh
python apache_beam/examples/complete/autocomplete_test.py
python setup.py nosetests
{toxinidir}/scripts/run_tox_cleanup.sh

[testenv:py36]
setenv =
RUN_SKIPPED_PY3_TESTS=0
commands =
Expand All @@ -78,7 +89,7 @@ commands =
python setup.py nosetests
{toxinidir}/scripts/run_tox_cleanup.sh

[testenv:py3-cython]
[testenv:py35-cython]
# cython tests are only expected to work in linux (2.x and 3.x)
# If we want to add other platforms in the future, it should be:
# `platform = linux2|darwin|...`
Expand All @@ -104,7 +115,7 @@ commands =
python setup.py nosetests
{toxinidir}/scripts/run_tox_cleanup.sh

[testenv:py3-gcp]
[testenv:py35-gcp]
setenv =
RUN_SKIPPED_PY3_TESTS=0
extras = test,gcp
Expand Down Expand Up @@ -143,7 +154,7 @@ commands =
time {toxinidir}/scripts/run_pylint_2to3.sh


[testenv:py3-lint]
[testenv:py35-lint]
deps =
pycodestyle==2.3.1
pylint==2.1.1
Expand Down