From 1d84d930d3f7fea73b1e610f9878c3a451b0647a Mon Sep 17 00:00:00 2001 From: Mark Liu Date: Mon, 11 Feb 2019 16:59:52 -0800 Subject: [PATCH] Fix PostCommit Python 3 build failure --- .../jon_PostCommit_Python3_Verify.groovy | 38 +++++++++++++++++++ build.gradle | 5 +++ .../beam/gradle/BeamModulePlugin.groovy | 2 +- sdks/python/build.gradle | 4 +- .../dataflow/build.gradle | 0 .../dataflow/py3/build.gradle | 2 +- .../direct/py3/build.gradle | 2 +- settings.gradle | 12 +++--- 8 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 .test-infra/jenkins/jon_PostCommit_Python3_Verify.groovy rename sdks/python/{precommit => test-suites}/dataflow/build.gradle (100%) rename sdks/python/{precommit => test-suites}/dataflow/py3/build.gradle (96%) rename sdks/python/{precommit => test-suites}/direct/py3/build.gradle (96%) diff --git a/.test-infra/jenkins/jon_PostCommit_Python3_Verify.groovy b/.test-infra/jenkins/jon_PostCommit_Python3_Verify.groovy new file mode 100644 index 000000000000..1eb1556e31d3 --- /dev/null +++ b/.test-infra/jenkins/jon_PostCommit_Python3_Verify.groovy @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import CommonJobProperties as commonJobProperties +import PostcommitJobBuilder + +// This job defines the Python postcommit tests. +PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python3_Verify', 'Run Python3 PostCommit', + 'Python SDK PostCommit Tests on Python 3', this) { + description('Runs postcommit tests on the Python SDK on Python 3.') + + // Set common parameters. + commonJobProperties.setTopLevelMainJobProperties(delegate) + + // Execute shell command to test Python SDK. + steps { + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(':python3PostCommit') + commonJobProperties.setGradleSwitches(delegate) + } + } +} diff --git a/build.gradle b/build.gradle index 23373a5badbb..9d24315efc2c 100644 --- a/build.gradle +++ b/build.gradle @@ -197,6 +197,11 @@ task pythonPostCommit() { dependsOn ":beam-sdks-python:postCommit" } +task python3PostCommit() { + dependsOn ":beam-sdks-python-test-suites-dataflow-py3:postCommitIT" + dependsOn ":beam-sdks-python-test-suites-direct-py3:postCommitIT" +} + task portablePythonPreCommit() { dependsOn ":beam-sdks-python:portablePreCommit" } diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 809381944022..4c58acf33a04 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1602,7 +1602,7 @@ class BeamModulePlugin implements Plugin { doLast { project.exec { executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && python ${pythonRootDir}/setup.py sdist --formats zip,gztar --dist-dir ${project.buildDir}" + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && python setup.py sdist --keep-temp --formats zip,gztar --dist-dir ${project.buildDir}" } def collection = project.fileTree("${project.buildDir}"){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz'} println "sdist archive name: ${collection.singleFile}" diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index e5d301d5f9b1..4421b800621a 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -111,8 +111,7 @@ task preCommit() { dependsOn "testPy2Gcp" dependsOn "testPy3Gcp" dependsOn "lint" - dependsOn ":beam-sdks-python-precommit-dataflow:preCommitIT" - dependsOn ":beam-sdks-python-precommit-direct-py3:preCommitIT" + dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT" } task portablePreCommit() { @@ -388,7 +387,6 @@ task postCommit() { dependsOn "directRunnerIT" dependsOn "hdfsIntegrationTest" dependsOn "postCommitIT" - dependsOn ":beam-sdks-python-precommit-dataflow-py3:preCommitIT" } diff --git a/sdks/python/precommit/dataflow/build.gradle b/sdks/python/test-suites/dataflow/build.gradle similarity index 100% rename from sdks/python/precommit/dataflow/build.gradle rename to sdks/python/test-suites/dataflow/build.gradle diff --git a/sdks/python/precommit/dataflow/py3/build.gradle b/sdks/python/test-suites/dataflow/py3/build.gradle similarity index 96% rename from sdks/python/precommit/dataflow/py3/build.gradle rename to sdks/python/test-suites/dataflow/py3/build.gradle index 9d343d5fe130..26ab0f1c29c8 100644 --- a/sdks/python/precommit/dataflow/py3/build.gradle +++ b/sdks/python/test-suites/dataflow/py3/build.gradle @@ -24,7 +24,7 @@ project.ext.python3 = true def runScriptsDir = "${project.rootDir}/sdks/python/scripts" -task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) { +task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { doLast { def tests = [ "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it", diff --git a/sdks/python/precommit/direct/py3/build.gradle b/sdks/python/test-suites/direct/py3/build.gradle similarity index 96% rename from sdks/python/precommit/direct/py3/build.gradle rename to sdks/python/test-suites/direct/py3/build.gradle index 7dc8a8259d9e..3e362adb981c 100644 --- a/sdks/python/precommit/direct/py3/build.gradle +++ b/sdks/python/test-suites/direct/py3/build.gradle @@ -24,7 +24,7 @@ project.ext.python3 = true def runScriptsDir = "${project.rootDir}/sdks/python/scripts" -task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) { +task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { // Run IT tests with TestDirectRunner in batch in Python 3. doLast { def batchTests = [ diff --git a/settings.gradle b/settings.gradle index f62eae305e79..53ee9a7222ea 100644 --- a/settings.gradle +++ b/settings.gradle @@ -205,12 +205,12 @@ include "beam-sdks-python-container" project(":beam-sdks-python-container").dir = file("sdks/python/container") include "beam-sdks-python-container-py3" project(":beam-sdks-python-container-py3").dir = file("sdks/python/container/py3") -include "beam-sdks-python-precommit-dataflow" -project(":beam-sdks-python-precommit-dataflow").dir = file("sdks/python/precommit/dataflow") -include "beam-sdks-python-precommit-dataflow-py3" -project(":beam-sdks-python-precommit-dataflow-py3").dir = file("sdks/python/precommit/dataflow/py3") -include "beam-sdks-python-precommit-direct-py3" -project(":beam-sdks-python-precommit-direct-py3").dir = file("sdks/python/precommit/direct/py3") +include "beam-sdks-python-test-suites-dataflow" +project(":beam-sdks-python-test-suites-dataflow").dir = file("sdks/python/test-suites/dataflow") +include "beam-sdks-python-test-suites-dataflow-py3" +project(":beam-sdks-python-test-suites-dataflow-py3").dir = file("sdks/python/test-suites/dataflow/py3") +include "beam-sdks-python-test-suites-direct-py3" +project(":beam-sdks-python-test-suites-direct-py3").dir = file("sdks/python/test-suites/direct/py3") include "beam-vendor-grpc-1_13_1" project(":beam-vendor-grpc-1_13_1").dir = file("vendor/grpc-1_13_1") include "beam-sdks-java-test-utils"