diff --git a/build.gradle b/build.gradle index e34eaa2fad75..4c47a26bb6a9 100644 --- a/build.gradle +++ b/build.gradle @@ -191,10 +191,7 @@ task goIntegrationTests() { } task pythonPreCommit() { - dependsOn ":beam-sdks-python:preCommitPy2" - dependsOn ":beam-sdks-python-test-suites-tox-py35:preCommitPy35" - dependsOn ":beam-sdks-python-test-suites-tox-py36:preCommitPy36" - dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT" + dependsOn ":beam-sdks-python:preCommit" } task pythonPostCommit() { diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index beb27e6e6d7c..7c50a52a8da5 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1611,38 +1611,16 @@ class BeamModulePlugin implements Plugin { outputs.dirs(project.ext.envdir) } - def pythonSdkDeps = project.files( - project.fileTree( - dir: "${project.rootDir}", - include: ['model/**', 'sdks/python/**'], - // Exclude temporary directories used in build and test. - exclude: [ - 'sdks/python/build/**', - 'sdks/python/dist/**', - 'sdks/python/target/**', - 'sdks/python/test-suites/**', - ]) - ) - def copiedSrcRoot = "${project.buildDir}/srcs" - project.configurations { distConfig } project.task('sdist', dependsOn: 'setupVirtualenv') { doLast { - // Copy sdk sources to an isolated directory - project.copy { - from pythonSdkDeps - into copiedSrcRoot - } - - // Build artifact project.exec { executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedSrcRoot}/sdks/python && python setup.py sdist --formats zip,gztar --dist-dir ${project.buildDir}" + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && python setup.py sdist --keep-temp --formats zip,gztar --dist-dir ${project.buildDir}" } def collection = project.fileTree("${project.buildDir}"){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz'} println "sdist archive name: ${collection.singleFile}" - // we need a fixed name for the artifact project.copy { from collection.singleFile; into "${project.buildDir}"; rename { 'apache-beam.tar.gz' } } } @@ -1668,7 +1646,7 @@ class BeamModulePlugin implements Plugin { project.exec { executable 'sh' args '-c', "if [ -e ${activate} ]; then " + - ". ${activate} && cd ${pythonRootDir} && python setup.py clean; " + + ". ${activate} && python ${pythonRootDir}/setup.py clean; " + "fi" } project.delete project.buildDir // Gradle build directory @@ -1695,21 +1673,6 @@ class BeamModulePlugin implements Plugin { } return argList.join(' ') } - - project.ext.toxTask = { name, tox_env -> - project.tasks.create(name) { - dependsOn = ['sdist'] - doLast { - def copiedPyRoot = "${copiedSrcRoot}/sdks/python" - project.exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env ${project.buildDir}/apache-beam.tar.gz" - } - } - inputs.files pythonSdkDeps - outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/") - } - } } } } diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index f13eb636a548..b83e614d79e9 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -38,8 +38,34 @@ build.dependsOn buildPython /*************************************************************************************************/ -// Unit tests for Python 2 -// See Python 3 tests in test-suites/tox +// Unit testing + +def pythonSdkDeps = files( + fileTree(dir: 'apache_beam', includes: ['**/*.py', '**/*.pyx', '**/*.pxd']), + fileTree(dir: 'apache_beam/testing/data'), + fileTree(dir: "${project.rootDir}/model"), + fileTree(dir: 'scripts'), + ".pylintrc", + "MANIFEST.in", + "gen_protos.py", + "setup.cfg", + "setup.py", + "test_config.py", + "tox.ini") + +def toxTask = { + name, tox_env -> tasks.create(name) { + dependsOn = ['setupVirtualenv'] + doLast { + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_tox.sh $tox_env" + } + } + inputs.files pythonSdkDeps + outputs.files fileTree(dir: "${project.rootDir}/sdks/python/target/.tox/${tox_env}/log/") + } +} task lint {} check.dependsOn lint @@ -50,12 +76,24 @@ lint.dependsOn lintPy27 toxTask "lintPy27_3", "py27-lint3" lint.dependsOn lintPy27_3 +toxTask "lintPy35", "py35-lint" +lint.dependsOn lintPy35 + toxTask "testPy2Gcp", "py27-gcp" test.dependsOn testPy2Gcp +toxTask "testPy35Gcp", "py35-gcp" +test.dependsOn testPy35Gcp + toxTask "testPython2", "py27" test.dependsOn testPython2 +toxTask "testPython35", "py35" +test.dependsOn testPython35 + +toxTask "testPython36", "py36" +test.dependsOn testPython36 + toxTask "testPy2Cython", "py27-cython" test.dependsOn testPy2Cython // Ensure that testPy2Cython runs exclusively to other tests. This line is not @@ -63,17 +101,29 @@ test.dependsOn testPy2Cython // project. testPy2Cython.mustRunAfter testPython2, testPy2Gcp +toxTask "testPy35Cython", "py35-cython" +test.dependsOn testPy35Cython +// Ensure that testPy35Cython runs exclusively to other tests. This line is not +// actually required, since gradle doesn't do parallel execution within a +// project. +testPy35Cython.mustRunAfter testPython35, testPy35Gcp + toxTask "docs", "docs" assemble.dependsOn docs toxTask "cover", "cover" -task preCommitPy2() { +task preCommit() { dependsOn "docs" dependsOn "testPy2Cython" + dependsOn "testPy35Cython" dependsOn "testPython2" + dependsOn "testPython35" + dependsOn "testPython36" dependsOn "testPy2Gcp" + dependsOn "testPy35Gcp" dependsOn "lint" + dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT" } task portablePreCommit() { diff --git a/sdks/python/scripts/generate_pydoc.sh b/sdks/python/scripts/generate_pydoc.sh index a9bffe3fbb2a..4929c5eb01fc 100755 --- a/sdks/python/scripts/generate_pydoc.sh +++ b/sdks/python/scripts/generate_pydoc.sh @@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ $PWD != *sdks/python ]]; then +if [[ "*sdks/python" != $PWD ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh index f0c01edddc03..273f96310ed8 100755 --- a/sdks/python/scripts/run_integration_test.sh +++ b/sdks/python/scripts/run_integration_test.sh @@ -163,7 +163,7 @@ if [[ -z $PIPELINE_OPTS ]]; then fi # Go to the Apache Beam Python SDK root - if [[ $PWD != *sdks/python ]]; then + if [[ "*sdks/python" != $PWD ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/scripts/run_mini_py3lint.sh b/sdks/python/scripts/run_mini_py3lint.sh index 27ca3ceff06b..0729c7ba2cfc 100755 --- a/sdks/python/scripts/run_mini_py3lint.sh +++ b/sdks/python/scripts/run_mini_py3lint.sh @@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ $PWD != *sdks/python ]]; then +if [[ "*sdks/python" != $PWD ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/scripts/run_pylint.sh b/sdks/python/scripts/run_pylint.sh index 2814b4f5c5d0..17f6aa706c1c 100755 --- a/sdks/python/scripts/run_pylint.sh +++ b/sdks/python/scripts/run_pylint.sh @@ -30,7 +30,7 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ $PWD != *sdks/python ]]; then +if [[ "*sdks/python" != $PWD ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/scripts/run_pylint_2to3.sh b/sdks/python/scripts/run_pylint_2to3.sh index 0853b09b3216..2957dd978fc0 100755 --- a/sdks/python/scripts/run_pylint_2to3.sh +++ b/sdks/python/scripts/run_pylint_2to3.sh @@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ $PWD != *sdks/python ]]; then +if [[ "*sdks/python" != $PWD ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/scripts/run_tox.sh b/sdks/python/scripts/run_tox.sh index e7990425e258..24ccb205913e 100755 --- a/sdks/python/scripts/run_tox.sh +++ b/sdks/python/scripts/run_tox.sh @@ -24,10 +24,9 @@ ########################################################################### # Usage check. -if [[ $# < 1 || $# > 2 ]]; then - printf "Usage: \n$> ./scripts/run_tox.sh []" +if [[ $# != 1 ]]; then + printf "Usage: \n$> ./scripts/run_tox.sh " printf "\n\ttox_environment: [required] Tox environment to run the test in.\n" - printf "\n\tsdk_location: [optional] SDK tarball artifact location.\n" exit 1 fi @@ -38,19 +37,11 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ $PWD != *sdks/python ]]; then +if [[ "*sdks/python" != $PWD ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi -# Used in tox.ini to isolate toxworkdir of each environment. -export ENV_NAME=.tox-$1 - -if [[ ! -z $2 ]]; then - tox -c tox.ini --recreate -e $1 --installpkg $2 -else - tox -c tox.ini --recreate -e $1 -fi - +tox -c tox.ini --recreate -e $1 exit_code=$? # Retry once for the specific exit code 245. if [[ $exit_code == 245 ]]; then diff --git a/sdks/python/scripts/run_tox_cleanup.sh b/sdks/python/scripts/run_tox_cleanup.sh index be4409525b53..87081da8e17b 100755 --- a/sdks/python/scripts/run_tox_cleanup.sh +++ b/sdks/python/scripts/run_tox_cleanup.sh @@ -26,7 +26,7 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ $PWD != *sdks/python ]]; then +if [[ "*sdks/python" != $PWD ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/test-suites/tox/py35/build.gradle b/sdks/python/test-suites/tox/py35/build.gradle deleted file mode 100644 index 5f86fe97c7c0..000000000000 --- a/sdks/python/test-suites/tox/py35/build.gradle +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Unit tests for Python 3.5 - */ - -plugins { id 'org.apache.beam.module' } -applyPythonNature() - -// Required to setup a Python 3 virtualenv. -project.ext.python3 = true - -task lint {} -check.dependsOn lint - -toxTask "lintPy35", "py35-lint" -lint.dependsOn lintPy35 - -toxTask "testPython35", "py35" -test.dependsOn testPython35 - -toxTask "testPy35Gcp", "py35-gcp" -test.dependsOn testPy35Gcp - -toxTask "testPy35Cython", "py35-cython" -test.dependsOn testPy35Cython -// Ensure that testPy35Cython runs exclusively to other tests. This line is not -// actually required, since gradle doesn't do parallel execution within a -// project. -testPy35Cython.mustRunAfter testPython35, testPy35Gcp - -task preCommitPy35() { - dependsOn "testPython35" - dependsOn "testPy35Gcp" - dependsOn "testPy35Cython" - dependsOn "lint" -} diff --git a/sdks/python/test-suites/tox/py36/build.gradle b/sdks/python/test-suites/tox/py36/build.gradle deleted file mode 100644 index 1bdfb786af4b..000000000000 --- a/sdks/python/test-suites/tox/py36/build.gradle +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Unit tests for Python 3.6 - */ - -plugins { id 'org.apache.beam.module' } -applyPythonNature() - -// Required to setup a Python 3 virtualenv. -project.ext.python3 = true - -toxTask "testPython36", "py36" -test.dependsOn testPython36 - -task preCommitPy36() { - dependsOn "testPython36" -} diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 2b765f8e412f..cd19a6b3b1c5 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -18,7 +18,7 @@ [tox] # new environments will be excluded by default unless explicitly added to envlist. envlist = py27,py35,py36,py27-{gcp,cython,lint,lint3},py35-{gcp,cython,lint},docs -toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox} +toxworkdir = {toxinidir}/target/.tox [pycodestyle] # Disable all errors and warnings except for the ones related to blank lines. diff --git a/settings.gradle b/settings.gradle index c453272b8587..b8fc6337373c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -207,10 +207,6 @@ include "beam-sdks-python-test-suites-dataflow-py3" project(":beam-sdks-python-test-suites-dataflow-py3").dir = file("sdks/python/test-suites/dataflow/py3") include "beam-sdks-python-test-suites-direct-py3" project(":beam-sdks-python-test-suites-direct-py3").dir = file("sdks/python/test-suites/direct/py3") -include "beam-sdks-python-test-suites-tox-py35" -project(":beam-sdks-python-test-suites-tox-py35").dir = file("sdks/python/test-suites/tox/py35") -include "beam-sdks-python-test-suites-tox-py36" -project(":beam-sdks-python-test-suites-tox-py36").dir = file("sdks/python/test-suites/tox/py36") include "beam-sdks-python-load-tests" project(":beam-sdks-python-load-tests").dir = file("sdks/python/apache_beam/testing/load_tests") include "beam-vendor-grpc-1_13_1"