From 996bc6c93aa8a71dbb5014ea4182ee39d2794d16 Mon Sep 17 00:00:00 2001 From: Mark Liu Date: Tue, 22 Jan 2019 14:20:09 -0800 Subject: [PATCH] Parallel Py3 tox tests --- build.gradle | 8 +- .../beam/gradle/BeamModulePlugin.groovy | 33 +++++++ sdks/python/build.gradle | 35 -------- sdks/python/precommit/py3/build.gradle | 38 ++++++++ sdks/python/precommit/py3/tox.ini | 90 +++++++++++++++++++ sdks/python/scripts/run_tox.sh | 8 +- sdks/python/tox.ini | 29 ------ settings.gradle | 2 + 8 files changed, 175 insertions(+), 68 deletions(-) create mode 100644 sdks/python/precommit/py3/build.gradle create mode 100644 sdks/python/precommit/py3/tox.ini diff --git a/build.gradle b/build.gradle index bb9de48e4f32..c4b5c5be6fe0 100644 --- a/build.gradle +++ b/build.gradle @@ -243,8 +243,14 @@ task goIntegrationTests() { dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" } -task pythonPreCommit() { +task pythonTox() { dependsOn ":beam-sdks-python:preCommit" + dependsOn ":beam-sdks-python-precommit-py3:preCommit" + finalizedBy ":beam-sdks-python:testCython" +} + +task pythonPreCommit() { + dependsOn pythonTox dependsOn ":beam-sdks-python-precommit-dataflow:precommitIT" } diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 7340f8f17de3..9b94dd050c39 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1605,9 +1605,42 @@ class BeamModulePlugin implements Plugin { args '-c', ". ${project.ext.envdir}/bin/activate && python ${project.ext.pythonRootDir}/setup.py clean" } project.delete project.buildDir + // project.delete "$project.path/target" + project.delete project.ext.envdir } } project.clean.dependsOn project.cleanPython + + def pythonSdkDeps = project.files( + project.fileTree(dir: "${project.ext.pythonRootDir}/apache_beam", includes: [ + '**/*.py', + '**/*.pyx', + '**/*.pxd' + ]), + project.fileTree(dir: "${project.ext.pythonRootDir}/apache_beam/testing/data"), + project.fileTree(dir: "${project.rootDir}/model"), + project.fileTree(dir: "${project.ext.pythonRootDir}/scripts"), + "${project.ext.pythonRootDir}/.pylintrc", + "${project.ext.pythonRootDir}/MANIFEST.in", + "${project.ext.pythonRootDir}/gen_protos.py", + "${project.ext.pythonRootDir}/setup.cfg", + "${project.ext.pythonRootDir}/setup.py", + "${project.ext.pythonRootDir}/test_config.py", + "${project.ext.pythonRootDir}/tox.ini") + + project.ext.toxTask = { name, tox_env, ini_path='' -> + project.tasks.create(name) { + dependsOn = ['setupVirtualenv'] + doLast { + project.exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && ${project.ext.pythonRootDir}/scripts/run_tox.sh $tox_env $ini_path" + } + } + inputs.files pythonSdkDeps + outputs.files project.fileTree(dir: "${project.ext.pythonRootDir}/target/.tox/${tox_env}/log/") + } + } } } } diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index f29c7286f13f..783ddd991e26 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -40,33 +40,6 @@ build.dependsOn buildPython /*************************************************************************************************/ // Unit testing -def pythonSdkDeps = files( - fileTree(dir: 'apache_beam', includes: ['**/*.py', '**/*.pyx', '**/*.pxd']), - fileTree(dir: 'apache_beam/testing/data'), - fileTree(dir: "${project.rootDir}/model"), - fileTree(dir: 'scripts'), - ".pylintrc", - "MANIFEST.in", - "gen_protos.py", - "setup.cfg", - "setup.py", - "test_config.py", - "tox.ini") - -def toxTask = { - name, tox_env -> tasks.create(name) { - dependsOn = ['setupVirtualenv'] - doLast { - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_tox.sh $tox_env" - } - } - inputs.files pythonSdkDeps - outputs.files fileTree(dir: "${project.rootDir}/sdks/python/target/.tox/${tox_env}/log/") - } -} - task lint {} check.dependsOn lint @@ -76,18 +49,12 @@ lint.dependsOn lintPy27 toxTask "lintPy27_3", "py27-lint3" lint.dependsOn lintPy27_3 -toxTask "lintPy3", "py3-lint" -lint.dependsOn lintPy3 - toxTask "testGcp", "py27-gcp" test.dependsOn testGcp toxTask "testPython2", "py27" test.dependsOn testPython2 -toxTask "testPython3", "py3" -test.dependsOn testPython3 - toxTask "testCython", "py27-cython" test.dependsOn testCython // Ensure that testCython runs exclusively to other tests. This line is not @@ -102,9 +69,7 @@ toxTask "cover", "cover" task preCommit() { dependsOn "docs" - dependsOn "testCython" dependsOn "testPython2" - dependsOn "testPython3" dependsOn "testGcp" dependsOn "lint" } diff --git a/sdks/python/precommit/py3/build.gradle b/sdks/python/precommit/py3/build.gradle new file mode 100644 index 000000000000..df71a8aa373f --- /dev/null +++ b/sdks/python/precommit/py3/build.gradle @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: org.apache.beam.gradle.BeamModulePlugin +applyPythonNature() + +task lint {} +check.dependsOn lint + +toxTask "lintPy3", "py3-lint", "precommit/py3/tox.ini" +lint.dependsOn lintPy3 + +toxTask "testPython3", "py3", "precommit/py3/tox.ini" +test.dependsOn testPython3 + +toxTask "testPython3Gcp", "py3-gcp", "precommit/py3/tox.ini" +test.dependsOn testPython3Gcp + +task preCommit() { + dependsOn "testPython3" + dependsOn "testPython3Gcp" + dependsOn "lint" +} diff --git a/sdks/python/precommit/py3/tox.ini b/sdks/python/precommit/py3/tox.ini new file mode 100644 index 000000000000..7aa9210ed8da --- /dev/null +++ b/sdks/python/precommit/py3/tox.ini @@ -0,0 +1,90 @@ +; +; Licensed to the Apache Software Foundation (ASF) under one or more +; contributor license agreements. See the NOTICE file distributed with +; this work for additional information regarding copyright ownership. +; The ASF licenses this file to You under the Apache License, Version 2.0 +; (the "License"); you may not use this file except in compliance with +; the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; + +[tox] +# new environments will be excluded by default unless explicitly added to envlist. +envlist = py3,py3-{gcp,lint} +toxworkdir = {toxinidir}/target/.tox +setupdir = {toxinidir}/../.. + +[pycodestyle] +# Disable all errors and warnings except for the ones related to blank lines. +# pylint does not check the number of blank lines. +select = E3 + +# Shared environment options. +[testenv] +# Set [] options for pip installation of apache-beam tarball. +extras = test +# Don't warn that these commands aren't installed. +whitelist_externals = + find + time +deps = + cython: cython==0.28.1 + future==0.16.0 +changedir = {toxinidir}/../.. + +# These 2 magic command overrides are required for Jenkins builds. +# Otherwise we get "OSError: [Errno 2] No such file or directory" errors. +# Source: +# https://github.com/tox-dev/tox/issues/123#issuecomment-284714629 +install_command = {envbindir}/python {envbindir}/pip install --retries 10 {opts} {packages} +list_dependencies_command = {envbindir}/python {envbindir}/pip freeze + +[testenv:py3] +setenv = + BEAM_EXPERIMENTAL_PY3=1 + RUN_SKIPPED_PY3_TESTS=0 +modules = + apache_beam.typehints,apache_beam.coders,apache_beam.options,apache_beam.tools,apache_beam.utils,apache_beam.internal,apache_beam.metrics,apache_beam.portability,apache_beam.pipeline_test,apache_beam.pvalue_test,apache_beam.runners,apache_beam.io.hadoopfilesystem_test,apache_beam.io.hdfs_integration_test,apache_beam.io.gcp.tests.utils_test,apache_beam.io.gcp.big_query_query_to_table_it_test,apache_beam.io.gcp.bigquery_io_read_it_test,apache_beam.io.gcp.bigquery_test,apache_beam.io.gcp.gcsfilesystem_test,apache_beam.io.gcp.gcsio_test,apache_beam.io.gcp.pubsub_integration_test,apache_beam.io.hdfs_integration_test,apache_beam.io.gcp.internal,apache_beam.io.filesystem_test,apache_beam.io.filesystems_test,apache_beam.io.range_trackers_test,apache_beam.io.sources_test,apache_beam.transforms,apache_beam.testing,apache_beam.io.filesystemio_test,apache_beam.io.localfilesystem_test,apache_beam.io.range_trackers_test,apache_beam.io.restriction_trackers_test,apache_beam.io.source_test_utils_test,apache_beam.io.concat_source_test,apache_beam.io.filebasedsink_test,apache_beam.io.filebasedsource_test,apache_beam.io.textio_test,apache_beam.io.tfrecordio_test +commands = + python --version + pip --version + {changedir}/scripts/run_tox_cleanup.sh + python {changedir}/setup.py nosetests --tests {[testenv:py3]modules} + {changedir}/scripts/run_tox_cleanup.sh + +[testenv:py3-gcp] +extras = test,gcp +setenv = + BEAM_EXPERIMENTAL_PY3=1 + RUN_SKIPPED_PY3_TESTS=0 +modules = + apache_beam.typehints +commands = + python --version + pip --version + {changedir}/scripts/run_tox_cleanup.sh + python {changedir}/apache_beam/examples/complete/autocomplete_test.py + python {changedir}/setup.py nosetests --tests {[testenv:py3-gcp]modules} + {changedir}/scripts/run_tox_cleanup.sh + +[testenv:py3-lint] +deps = + pycodestyle==2.3.1 + pylint==2.1.1 + future==0.16.0 + isort==4.2.15 + flake8==3.5.0 +setenv = + BEAM_EXPERIMENTAL_PY3=1 +commands = + pylint --version + pip --version + {changedir}/scripts/run_tox_cleanup.sh + time {changedir}/scripts/run_mini_py3lint.sh diff --git a/sdks/python/scripts/run_tox.sh b/sdks/python/scripts/run_tox.sh index 24ccb205913e..4e9ee5d84f53 100755 --- a/sdks/python/scripts/run_tox.sh +++ b/sdks/python/scripts/run_tox.sh @@ -24,8 +24,8 @@ ########################################################################### # Usage check. -if [[ $# != 1 ]]; then - printf "Usage: \n$> ./scripts/run_tox.sh " +if [[ $# < 1 || $# > 3 ]]; then + printf "Usage: \n$> ./scripts/run_tox.sh []" printf "\n\ttox_environment: [required] Tox environment to run the test in.\n" exit 1 fi @@ -41,7 +41,9 @@ if [[ "*sdks/python" != $PWD ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi -tox -c tox.ini --recreate -e $1 +ini_path=${2:-'tox.ini'} + +tox -c $ini_path --recreate -e $1 exit_code=$? # Retry once for the specific exit code 245. if [[ $exit_code == 245 ]]; then diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 75f3fc02159a..dd5c5e978cca 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -53,19 +53,6 @@ commands = python setup.py nosetests {toxinidir}/scripts/run_tox_cleanup.sh -[testenv:py3] -setenv = - BEAM_EXPERIMENTAL_PY3=1 - RUN_SKIPPED_PY3_TESTS=0 -modules = - apache_beam.typehints,apache_beam.coders,apache_beam.options,apache_beam.tools,apache_beam.utils,apache_beam.internal,apache_beam.metrics,apache_beam.portability,apache_beam.pipeline_test,apache_beam.pvalue_test,apache_beam.runners,apache_beam.io.hadoopfilesystem_test,apache_beam.io.hdfs_integration_test,apache_beam.io.gcp.tests.utils_test,apache_beam.io.gcp.big_query_query_to_table_it_test,apache_beam.io.gcp.bigquery_io_read_it_test,apache_beam.io.gcp.bigquery_test,apache_beam.io.gcp.gcsfilesystem_test,apache_beam.io.gcp.gcsio_test,apache_beam.io.gcp.pubsub_integration_test,apache_beam.io.hdfs_integration_test,apache_beam.io.gcp.internal,apache_beam.io.filesystem_test,apache_beam.io.filesystems_test,apache_beam.io.range_trackers_test,apache_beam.io.sources_test,apache_beam.transforms,apache_beam.testing,apache_beam.io.filesystemio_test,apache_beam.io.localfilesystem_test,apache_beam.io.range_trackers_test,apache_beam.io.restriction_trackers_test,apache_beam.io.source_test_utils_test,apache_beam.io.concat_source_test,apache_beam.io.filebasedsink_test,apache_beam.io.filebasedsource_test,apache_beam.io.textio_test,apache_beam.io.tfrecordio_test -commands = - python --version - pip --version - {toxinidir}/scripts/run_tox_cleanup.sh - python setup.py nosetests --tests {[testenv:py3]modules} - {toxinidir}/scripts/run_tox_cleanup.sh - [testenv:py27-cython] # cython tests are only expected to work in linux (2.x and 3.x) # If we want to add other platforms in the future, it should be: @@ -117,22 +104,6 @@ commands = {toxinidir}/scripts/run_tox_cleanup.sh time {toxinidir}/scripts/run_pylint_2to3.sh - -[testenv:py3-lint] -deps = - pycodestyle==2.3.1 - pylint==2.1.1 - future==0.16.0 - isort==4.2.15 - flake8==3.5.0 -setenv = - BEAM_EXPERIMENTAL_PY3=1 -commands = - pylint --version - pip --version - {toxinidir}/scripts/run_tox_cleanup.sh - time {toxinidir}/scripts/run_mini_py3lint.sh - [testenv:docs] extras = test,gcp,docs deps = diff --git a/settings.gradle b/settings.gradle index 7d44f83772c0..b523efd18671 100644 --- a/settings.gradle +++ b/settings.gradle @@ -207,6 +207,8 @@ include "beam-sdks-python-container-py3" project(":beam-sdks-python-container-py3").dir = file("sdks/python/container/py3") include "beam-sdks-python-precommit-dataflow" project(":beam-sdks-python-precommit-dataflow").dir = file("sdks/python/precommit/dataflow") +include "beam-sdks-python-precommit-py3" +project(":beam-sdks-python-precommit-py3").dir = file("sdks/python/precommit/py3") include "beam-vendor-grpc-1_13_1" project(":beam-vendor-grpc-1_13_1").dir = file("vendor/grpc-1_13_1") include "beam-sdks-java-test-utils"