diff --git a/.test-infra/jenkins/job_Release_NightlySnapshot.groovy b/.test-infra/jenkins/job_Release_NightlySnapshot.groovy index 415db871325e..1a925e198566 100644 --- a/.test-infra/jenkins/job_Release_NightlySnapshot.groovy +++ b/.test-infra/jenkins/job_Release_NightlySnapshot.groovy @@ -28,7 +28,7 @@ job('beam_Release_NightlySnapshot') { concurrentBuild() // Set common parameters. Timeout is longer, to avoid [BEAM-5774]. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 140) + commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 200) // This is a post-commit job that runs once per day, not for every push. commonJobProperties.setAutoJob( diff --git a/build.gradle b/build.gradle index 4c47a26bb6a9..e34eaa2fad75 100644 --- a/build.gradle +++ b/build.gradle @@ -191,7 +191,10 @@ task goIntegrationTests() { } task pythonPreCommit() { - dependsOn ":beam-sdks-python:preCommit" + dependsOn ":beam-sdks-python:preCommitPy2" + dependsOn ":beam-sdks-python-test-suites-tox-py35:preCommitPy35" + dependsOn ":beam-sdks-python-test-suites-tox-py36:preCommitPy36" + dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT" } task pythonPostCommit() { diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 7c50a52a8da5..985ed979f384 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1611,16 +1611,38 @@ class BeamModulePlugin implements Plugin { outputs.dirs(project.ext.envdir) } + def pythonSdkDeps = project.files( + project.fileTree( + dir: "${project.rootDir}", + include: ['model/**', 'sdks/python/**'], + // Exclude temporary directories used in build and test. + exclude: [ + '**/build/**', + '**/dist/**', + '**/target/**', + 'sdks/python/test-suites/**', + ]) + ) + def copiedSrcRoot = "${project.buildDir}/srcs" + project.configurations { distConfig } project.task('sdist', dependsOn: 'setupVirtualenv') { doLast { + // Copy sdk sources to an isolated directory + project.copy { + from pythonSdkDeps + into copiedSrcRoot + } + + // Build artifact project.exec { executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && python setup.py sdist --keep-temp --formats zip,gztar --dist-dir ${project.buildDir}" + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedSrcRoot}/sdks/python && python setup.py sdist --formats zip,gztar --dist-dir ${project.buildDir}" } - def collection = project.fileTree("${project.buildDir}"){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz'} + def collection = project.fileTree("${project.buildDir}"){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz', 'srcs/**'} println "sdist archive name: ${collection.singleFile}" + // we need a fixed name for the artifact project.copy { from collection.singleFile; into "${project.buildDir}"; rename { 'apache-beam.tar.gz' } } } @@ -1646,7 +1668,7 @@ class BeamModulePlugin implements Plugin { project.exec { executable 'sh' args '-c', "if [ -e ${activate} ]; then " + - ". ${activate} && python ${pythonRootDir}/setup.py clean; " + + ". ${activate} && cd ${pythonRootDir} && python setup.py clean; " + "fi" } project.delete project.buildDir // Gradle build directory @@ -1673,6 +1695,21 @@ class BeamModulePlugin implements Plugin { } return argList.join(' ') } + + project.ext.toxTask = { name, tox_env -> + project.tasks.create(name) { + dependsOn = ['sdist'] + doLast { + def copiedPyRoot = "${copiedSrcRoot}/sdks/python" + project.exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env ${project.buildDir}/apache-beam.tar.gz" + } + } + inputs.files pythonSdkDeps + outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/") + } + } } } } diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index b83e614d79e9..f13eb636a548 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -38,34 +38,8 @@ build.dependsOn buildPython /*************************************************************************************************/ -// Unit testing - -def pythonSdkDeps = files( - fileTree(dir: 'apache_beam', includes: ['**/*.py', '**/*.pyx', '**/*.pxd']), - fileTree(dir: 'apache_beam/testing/data'), - fileTree(dir: "${project.rootDir}/model"), - fileTree(dir: 'scripts'), - ".pylintrc", - "MANIFEST.in", - "gen_protos.py", - "setup.cfg", - "setup.py", - "test_config.py", - "tox.ini") - -def toxTask = { - name, tox_env -> tasks.create(name) { - dependsOn = ['setupVirtualenv'] - doLast { - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_tox.sh $tox_env" - } - } - inputs.files pythonSdkDeps - outputs.files fileTree(dir: "${project.rootDir}/sdks/python/target/.tox/${tox_env}/log/") - } -} +// Unit tests for Python 2 +// See Python 3 tests in test-suites/tox task lint {} check.dependsOn lint @@ -76,24 +50,12 @@ lint.dependsOn lintPy27 toxTask "lintPy27_3", "py27-lint3" lint.dependsOn lintPy27_3 -toxTask "lintPy35", "py35-lint" -lint.dependsOn lintPy35 - toxTask "testPy2Gcp", "py27-gcp" test.dependsOn testPy2Gcp -toxTask "testPy35Gcp", "py35-gcp" -test.dependsOn testPy35Gcp - toxTask "testPython2", "py27" test.dependsOn testPython2 -toxTask "testPython35", "py35" -test.dependsOn testPython35 - -toxTask "testPython36", "py36" -test.dependsOn testPython36 - toxTask "testPy2Cython", "py27-cython" test.dependsOn testPy2Cython // Ensure that testPy2Cython runs exclusively to other tests. This line is not @@ -101,29 +63,17 @@ test.dependsOn testPy2Cython // project. testPy2Cython.mustRunAfter testPython2, testPy2Gcp -toxTask "testPy35Cython", "py35-cython" -test.dependsOn testPy35Cython -// Ensure that testPy35Cython runs exclusively to other tests. This line is not -// actually required, since gradle doesn't do parallel execution within a -// project. -testPy35Cython.mustRunAfter testPython35, testPy35Gcp - toxTask "docs", "docs" assemble.dependsOn docs toxTask "cover", "cover" -task preCommit() { +task preCommitPy2() { dependsOn "docs" dependsOn "testPy2Cython" - dependsOn "testPy35Cython" dependsOn "testPython2" - dependsOn "testPython35" - dependsOn "testPython36" dependsOn "testPy2Gcp" - dependsOn "testPy35Gcp" dependsOn "lint" - dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT" } task portablePreCommit() { diff --git a/sdks/python/scripts/generate_pydoc.sh b/sdks/python/scripts/generate_pydoc.sh index 4929c5eb01fc..a9bffe3fbb2a 100755 --- a/sdks/python/scripts/generate_pydoc.sh +++ b/sdks/python/scripts/generate_pydoc.sh @@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ "*sdks/python" != $PWD ]]; then +if [[ $PWD != *sdks/python ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh index 273f96310ed8..f0c01edddc03 100755 --- a/sdks/python/scripts/run_integration_test.sh +++ b/sdks/python/scripts/run_integration_test.sh @@ -163,7 +163,7 @@ if [[ -z $PIPELINE_OPTS ]]; then fi # Go to the Apache Beam Python SDK root - if [[ "*sdks/python" != $PWD ]]; then + if [[ $PWD != *sdks/python ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/scripts/run_mini_py3lint.sh b/sdks/python/scripts/run_mini_py3lint.sh index 0729c7ba2cfc..27ca3ceff06b 100755 --- a/sdks/python/scripts/run_mini_py3lint.sh +++ b/sdks/python/scripts/run_mini_py3lint.sh @@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ "*sdks/python" != $PWD ]]; then +if [[ $PWD != *sdks/python ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/scripts/run_pylint.sh b/sdks/python/scripts/run_pylint.sh index 17f6aa706c1c..2814b4f5c5d0 100755 --- a/sdks/python/scripts/run_pylint.sh +++ b/sdks/python/scripts/run_pylint.sh @@ -30,7 +30,7 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ "*sdks/python" != $PWD ]]; then +if [[ $PWD != *sdks/python ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/scripts/run_pylint_2to3.sh b/sdks/python/scripts/run_pylint_2to3.sh index 2957dd978fc0..0853b09b3216 100755 --- a/sdks/python/scripts/run_pylint_2to3.sh +++ b/sdks/python/scripts/run_pylint_2to3.sh @@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ "*sdks/python" != $PWD ]]; then +if [[ $PWD != *sdks/python ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/scripts/run_tox.sh b/sdks/python/scripts/run_tox.sh index 24ccb205913e..e7990425e258 100755 --- a/sdks/python/scripts/run_tox.sh +++ b/sdks/python/scripts/run_tox.sh @@ -24,9 +24,10 @@ ########################################################################### # Usage check. -if [[ $# != 1 ]]; then - printf "Usage: \n$> ./scripts/run_tox.sh " +if [[ $# < 1 || $# > 2 ]]; then + printf "Usage: \n$> ./scripts/run_tox.sh []" printf "\n\ttox_environment: [required] Tox environment to run the test in.\n" + printf "\n\tsdk_location: [optional] SDK tarball artifact location.\n" exit 1 fi @@ -37,11 +38,19 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ "*sdks/python" != $PWD ]]; then +if [[ $PWD != *sdks/python ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi -tox -c tox.ini --recreate -e $1 +# Used in tox.ini to isolate toxworkdir of each environment. +export ENV_NAME=.tox-$1 + +if [[ ! -z $2 ]]; then + tox -c tox.ini --recreate -e $1 --installpkg $2 +else + tox -c tox.ini --recreate -e $1 +fi + exit_code=$? # Retry once for the specific exit code 245. if [[ $exit_code == 245 ]]; then diff --git a/sdks/python/scripts/run_tox_cleanup.sh b/sdks/python/scripts/run_tox_cleanup.sh index 87081da8e17b..be4409525b53 100755 --- a/sdks/python/scripts/run_tox_cleanup.sh +++ b/sdks/python/scripts/run_tox_cleanup.sh @@ -26,7 +26,7 @@ if [[ $PWD != *sdks/python* ]]; then fi # Go to the Apache Beam Python SDK root -if [[ "*sdks/python" != $PWD ]]; then +if [[ $PWD != *sdks/python ]]; then cd $(pwd | sed 's/sdks\/python.*/sdks\/python/') fi diff --git a/sdks/python/test-suites/tox/py35/build.gradle b/sdks/python/test-suites/tox/py35/build.gradle new file mode 100644 index 000000000000..5f86fe97c7c0 --- /dev/null +++ b/sdks/python/test-suites/tox/py35/build.gradle @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests for Python 3.5 + */ + +plugins { id 'org.apache.beam.module' } +applyPythonNature() + +// Required to setup a Python 3 virtualenv. +project.ext.python3 = true + +task lint {} +check.dependsOn lint + +toxTask "lintPy35", "py35-lint" +lint.dependsOn lintPy35 + +toxTask "testPython35", "py35" +test.dependsOn testPython35 + +toxTask "testPy35Gcp", "py35-gcp" +test.dependsOn testPy35Gcp + +toxTask "testPy35Cython", "py35-cython" +test.dependsOn testPy35Cython +// Ensure that testPy35Cython runs exclusively to other tests. This line is not +// actually required, since gradle doesn't do parallel execution within a +// project. +testPy35Cython.mustRunAfter testPython35, testPy35Gcp + +task preCommitPy35() { + dependsOn "testPython35" + dependsOn "testPy35Gcp" + dependsOn "testPy35Cython" + dependsOn "lint" +} diff --git a/sdks/python/test-suites/tox/py36/build.gradle b/sdks/python/test-suites/tox/py36/build.gradle new file mode 100644 index 000000000000..8dc497d0e2f1 --- /dev/null +++ b/sdks/python/test-suites/tox/py36/build.gradle @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests for Python 3.6 + */ + +plugins { id 'org.apache.beam.module' } +applyPythonNature() + +toxTask "testPython36", "py36" +test.dependsOn testPython36 + +task preCommitPy36() { + dependsOn "testPython36" +} diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index cd19a6b3b1c5..2b765f8e412f 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -18,7 +18,7 @@ [tox] # new environments will be excluded by default unless explicitly added to envlist. envlist = py27,py35,py36,py27-{gcp,cython,lint,lint3},py35-{gcp,cython,lint},docs -toxworkdir = {toxinidir}/target/.tox +toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox} [pycodestyle] # Disable all errors and warnings except for the ones related to blank lines. diff --git a/settings.gradle b/settings.gradle index b8fc6337373c..c453272b8587 100644 --- a/settings.gradle +++ b/settings.gradle @@ -207,6 +207,10 @@ include "beam-sdks-python-test-suites-dataflow-py3" project(":beam-sdks-python-test-suites-dataflow-py3").dir = file("sdks/python/test-suites/dataflow/py3") include "beam-sdks-python-test-suites-direct-py3" project(":beam-sdks-python-test-suites-direct-py3").dir = file("sdks/python/test-suites/direct/py3") +include "beam-sdks-python-test-suites-tox-py35" +project(":beam-sdks-python-test-suites-tox-py35").dir = file("sdks/python/test-suites/tox/py35") +include "beam-sdks-python-test-suites-tox-py36" +project(":beam-sdks-python-test-suites-tox-py36").dir = file("sdks/python/test-suites/tox/py36") include "beam-sdks-python-load-tests" project(":beam-sdks-python-load-tests").dir = file("sdks/python/apache_beam/testing/load_tests") include "beam-vendor-grpc-1_13_1"