From 13450c430fffe463e7bb45fd7e6e2b97134c603d Mon Sep 17 00:00:00 2001 From: Mark Liu Date: Mon, 5 Aug 2019 10:58:40 -0700 Subject: [PATCH 1/3] [BEAM-6907] Reuse Python tarball in integration tests --- .../job_PerformanceTests_Python.groovy | 15 +------ .../beam/gradle/BeamModulePlugin.groovy | 43 ++++--------------- sdks/python/build.gradle | 26 +++++++++++ sdks/python/container/build.gradle | 2 +- sdks/python/container/py3/build.gradle | 2 +- .../test-suites/dataflow/py2/build.gradle | 24 ++++++----- .../test-suites/dataflow/py35/build.gradle | 17 +++++--- .../test-suites/dataflow/py36/build.gradle | 17 +++++--- .../test-suites/dataflow/py37/build.gradle | 21 +++++---- 9 files changed, 88 insertions(+), 79 deletions(-) diff --git a/.test-infra/jenkins/job_PerformanceTests_Python.groovy b/.test-infra/jenkins/job_PerformanceTests_Python.groovy index 686af9bcb062..131417f88201 100644 --- a/.test-infra/jenkins/job_PerformanceTests_Python.groovy +++ b/.test-infra/jenkins/job_PerformanceTests_Python.groovy @@ -41,7 +41,7 @@ class PerformanceTestConfigurations { String itModule // A benchmark defined flag, will pass to benchmark as "--beam_python_sdk_location". // It's the location of Python SDK distribution archive which is required for TestDataflowRunner. - String pythonSdkLocation = '' + String pythonSdkLocation = 'build/apache-beam.tar.gz' // A benchmark defined flag, will pass to benchmark as "--beam_runner" String runner = 'TestDataflowRunner' // A benchmark defined flag, will pass to benchmark as "--beam_it_timeout" @@ -152,8 +152,7 @@ private void createPythonPerformanceTestJob(PerformanceTestConfigurations testCo beam_it_class : testConfig.itClass, beam_it_module : testConfig.itModule, beam_prebuilt : 'true', // Python benchmark don't need to prebuild repo before running - beam_python_sdk_location: getSDKLocationFromModule(testConfig.pythonSdkLocation, - testConfig.itModule), + beam_python_sdk_location: testConfig.pythonSdkLocation, beam_runner : testConfig.runner, beam_it_timeout : testConfig.itTimeoutSec.toString(), beam_it_args : joinPipelineArgs(testConfig.extraPipelineArgs), @@ -172,13 +171,3 @@ private static String joinPipelineArgs(Map pipelineArgs) { }) return pipelineArgList.join(',') } - - -// Get relative path of sdk location based on itModule if the location is not provided. -private static String getSDKLocationFromModule(String pythonSDKLocation, String itModule) { - if (!pythonSDKLocation && itModule.startsWith(":sdks:python")) { - def projectPath = (itModule - ':sdks:python').replace(':', '/') - return (projectPath + "/build/apache-beam.tar.gz").substring(1) - } - return pythonSDKLocation -} diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 5cef87e3c47d..f1f60dad52e3 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1858,7 +1858,7 @@ class BeamModulePlugin implements Plugin { outputs.dirs(project.ext.envdir) } - def pythonSdkDeps = project.files( + project.ext.pythonSdkDeps = project.files( project.fileTree( dir: "${project.rootDir}", include: ['model/**', 'sdks/python/**'], @@ -1871,35 +1871,10 @@ class BeamModulePlugin implements Plugin { ]) ) def copiedSrcRoot = "${project.buildDir}/srcs" - def tarball = "apache-beam.tar.gz" - project.configurations { distConfig } - - project.task('sdist', dependsOn: 'setupVirtualenv') { - doLast { - // Copy sdk sources to an isolated directory - project.copy { - from pythonSdkDeps - into copiedSrcRoot - } - - // Build artifact - project.exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedSrcRoot}/sdks/python && python setup.py -q sdist --formats zip,gztar --dist-dir ${project.buildDir}" - } - def collection = project.fileTree("${project.buildDir}"){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz', 'srcs/**'} - - // we need a fixed name for the artifact - project.copy { from collection.singleFile; into "${project.buildDir}"; rename { tarball } } - } - inputs.files pythonSdkDeps - outputs.file "${project.buildDir}/${tarball}" - } - - project.artifacts { - distConfig file: project.file("${project.buildDir}/${tarball}"), builtBy: project.sdist - } + // Create new configuration distTarBall which represents Python source + // distribution tarball generated by :sdks:python:sdist. + project.configurations { distTarBall } project.task('installGcpTest', dependsOn: 'setupVirtualenv') { doLast { @@ -1909,7 +1884,6 @@ class BeamModulePlugin implements Plugin { } } } - project.installGcpTest.mustRunAfter project.sdist project.task('cleanPython') { doLast { @@ -1947,15 +1921,16 @@ class BeamModulePlugin implements Plugin { project.ext.toxTask = { name, tox_env -> project.tasks.create(name) { - dependsOn = ['sdist'] + dependsOn = [':sdks:python:sdist'] doLast { def copiedPyRoot = "${copiedSrcRoot}/sdks/python" + def distTarBall = "${pythonRootDir}/build/apache-beam.tar.gz" project.exec { executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env ${project.buildDir}/apache-beam.tar.gz" + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env $distTarBall" } } - inputs.files pythonSdkDeps + inputs.files project.pythonSdkDeps outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/") } } @@ -1969,7 +1944,7 @@ class BeamModulePlugin implements Plugin { project.task('integrationTest') { dependsOn 'installGcpTest' - dependsOn 'sdist' + dependsOn ':sdks:python:sdist' doLast { def argMap = [:] diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index 5fc3740d0332..f2134695b7f9 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -34,6 +34,32 @@ task buildPython(dependsOn: 'setupVirtualenv') { } build.dependsOn buildPython +// Create a Python source distribution tarball. +def tarball = "apache-beam.tar.gz" +task sdist { + dependsOn setupVirtualenv + + doLast { + // Build artifact + project.exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && python setup.py -q sdist --formats zip,gztar --dist-dir ${project.buildDir}" + } + + def collection = fileTree(project.buildDir){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz', 'srcs/**'} + + // we need a fixed name for the artifact + copy { from collection.singleFile; into project.buildDir; rename { tarball } } + println "Create distribution tar file ${tarball} in ${project.buildDir}" + } + inputs.files project.pythonSdkDeps + outputs.file "${project.buildDir}/${tarball}" +} + +artifacts { + distTarBall file: file("${project.buildDir}/${tarball}"), builtBy: sdist +} + /*************************************************************************************************/ // Non-testing builds and analysis tasks diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle index a6650e988975..b8aa84966b1e 100644 --- a/sdks/python/container/build.gradle +++ b/sdks/python/container/build.gradle @@ -39,7 +39,7 @@ dependencies { build name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir test name: './github.com/apache/beam/sdks/go', dir: project(':sdks:go').projectDir } - sdkSourceTarball project(path: ":sdks:python", configuration: "distConfig") + sdkSourceTarball project(path: ":sdks:python", configuration: "distTarBall") } task copyDockerfileDependencies(type: Copy, dependsOn: goBuild) { diff --git a/sdks/python/container/py3/build.gradle b/sdks/python/container/py3/build.gradle index 05710bc35866..37f1c881d484 100644 --- a/sdks/python/container/py3/build.gradle +++ b/sdks/python/container/py3/build.gradle @@ -30,7 +30,7 @@ configurations { } dependencies { - sdkSourceTarball project(path: ":sdks:python", configuration: "distConfig") + sdkSourceTarball project(path: ":sdks:python", configuration: "distTarBall") sdkHarnessLauncher project(path: ":sdks:python:container", configuration: "sdkHarnessLauncher") } diff --git a/sdks/python/test-suites/dataflow/py2/build.gradle b/sdks/python/test-suites/dataflow/py2/build.gradle index a872242392fe..bdf0c2bc4a73 100644 --- a/sdks/python/test-suites/dataflow/py2/build.gradle +++ b/sdks/python/test-suites/dataflow/py2/build.gradle @@ -20,6 +20,10 @@ plugins { id 'org.apache.beam.module' } applyPythonNature() enablePythonPerformanceTest() +dependencies { + distTarBall project(path: ":sdks:python", configuration: "distTarBall") +} + def runScriptsDir = "${project.rootDir}/sdks/python/scripts" // Basic test options for ITs running on Jenkins. @@ -29,7 +33,7 @@ def basicTestOpts = [ "--process-timeout=4500", // timeout of whole command execution ] -task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) { +task preCommitIT(dependsOn: [':sdks:python:sdist', 'installGcpTest']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -48,7 +52,7 @@ task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) { ] def cmdArgs = project.mapToArgString([ "test_opts": testOpts, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "preCommitIT-df" ]) @@ -61,7 +65,7 @@ task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) { } // Run PostCommit integration tests on default runner (TestDataflowRunner) -task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) { +task postCommitIT(dependsOn: ['installGcpTest', ':sdks:python:sdist']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -70,7 +74,7 @@ task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) { def testOpts = basicTestOpts + ["--attr=IT"] def cmdArgs = project.mapToArgString(["test_opts": testOpts, "worker_jar": dataflowWorkerJar, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "suite": "postCommitIT-df"]) exec { executable 'sh' @@ -79,7 +83,7 @@ task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) { } } -task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { +task validatesRunnerBatchTests(dependsOn: ['installGcpTest', ':sdks:python:sdist']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -88,7 +92,7 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { def testOpts = basicTestOpts + ["--attr=ValidatesRunner"] def cmdArgs = project.mapToArgString(["test_opts": testOpts, "worker_jar": dataflowWorkerJar, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "suite": "validatesRunnerBatchTests-df"]) exec { executable 'sh' @@ -97,7 +101,7 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { } } -task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { +task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', ':sdks:python:sdist']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -108,7 +112,7 @@ task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { def argMap = ["test_opts": testOpts, "streaming": "true", "worker_jar": dataflowWorkerJar, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "suite": "validatesRunnerStreamingTests-df"] def cmdArgs = project.mapToArgString(argMap) exec { @@ -118,11 +122,11 @@ task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { } } -task dataflowChicagoTaxiExample(dependsOn: ['installGcpTest', 'sdist']) { +task dataflowChicagoTaxiExample(dependsOn: ['installGcpTest', ':sdks:python:sdist']) { def gcsRoot = findProperty('gcsRoot') def runner = findProperty('runner') - def cliArgs = "${gcsRoot} ${runner} ${project.buildDir}/apache-beam.tar.gz" + def cliArgs = "${gcsRoot} ${runner} ${files(configurations.distTarBall.files).singleFile}" doLast { exec { diff --git a/sdks/python/test-suites/dataflow/py35/build.gradle b/sdks/python/test-suites/dataflow/py35/build.gradle index c81f6723ac21..813aaa11ba09 100644 --- a/sdks/python/test-suites/dataflow/py35/build.gradle +++ b/sdks/python/test-suites/dataflow/py35/build.gradle @@ -20,9 +20,14 @@ apply plugin: org.apache.beam.gradle.BeamModulePlugin applyPythonNature() enablePythonPerformanceTest() +dependencies { + distTarBall project(path: ":sdks:python", configuration: "distTarBall") +} + // Required to setup a Python 3 virtualenv. pythonVersion = '3.5' + def runScriptsDir = "${project.rootDir}/sdks/python/scripts" // Basic test options for ITs running on Jenkins. @@ -32,7 +37,7 @@ def basicTestOpts = [ "--process-timeout=4500", // timeout of whole command execution ] -task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { +task postCommitIT(dependsOn: [':sdks:python:sdist', 'installGcpTest']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -42,7 +47,7 @@ task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { def cmdArgs = project.mapToArgString([ "test_opts": testOpts, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "postCommitIT-df-py35" ]) @@ -53,7 +58,7 @@ task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { } } -task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { +task validatesRunnerBatchTests(dependsOn: ['installGcpTest', ':sdks:python:sdist']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -62,7 +67,7 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { def testOpts = basicTestOpts + ["--attr=ValidatesRunner"] def cmdArgs = project.mapToArgString([ "test_opts": testOpts, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "validatesRunnerBatchTests-df-py35" ]) @@ -73,7 +78,7 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { } } -task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { +task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', ':sdks:python:sdist']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -83,7 +88,7 @@ task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-streaming"] def argMap = ["test_opts": testOpts, "streaming": "true", - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "validatesRunnerStreamingTests-df-py35"] def cmdArgs = project.mapToArgString(argMap) diff --git a/sdks/python/test-suites/dataflow/py36/build.gradle b/sdks/python/test-suites/dataflow/py36/build.gradle index e09997c892f6..604a6d9247bd 100644 --- a/sdks/python/test-suites/dataflow/py36/build.gradle +++ b/sdks/python/test-suites/dataflow/py36/build.gradle @@ -20,9 +20,14 @@ apply plugin: org.apache.beam.gradle.BeamModulePlugin applyPythonNature() enablePythonPerformanceTest() +dependencies { + distTarBall project(path: ":sdks:python", configuration: "distTarBall") +} + // Required to setup a Python 3 virtualenv. pythonVersion = '3.6' + def runScriptsDir = "${project.rootDir}/sdks/python/scripts" // Basic test options for ITs running on Jenkins. @@ -32,7 +37,7 @@ def basicTestOpts = [ "--process-timeout=4500", // timeout of whole command execution ] -task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { +task postCommitIT(dependsOn: [':sdks:python:sdist', 'installGcpTest']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -42,7 +47,7 @@ task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { def cmdArgs = project.mapToArgString([ "test_opts": testOpts, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "postCommitIT-df-py36" ]) @@ -53,7 +58,7 @@ task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { } } -task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { +task validatesRunnerBatchTests(dependsOn: ['installGcpTest', ':sdks:python:sdist']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -62,7 +67,7 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { def testOpts = basicTestOpts + ["--attr=ValidatesRunner"] def cmdArgs = project.mapToArgString([ "test_opts": testOpts, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "validatesRunnerBatchTests-df-py36" ]) @@ -73,7 +78,7 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { } } -task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { +task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', ':sdks:python:sdist']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -83,7 +88,7 @@ task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-streaming"] def argMap = ["test_opts": testOpts, "streaming": "true", - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "validatesRunnerStreamingTests-df-py36"] def cmdArgs = project.mapToArgString(argMap) diff --git a/sdks/python/test-suites/dataflow/py37/build.gradle b/sdks/python/test-suites/dataflow/py37/build.gradle index b5764ceb102c..da4b2e54bcf8 100644 --- a/sdks/python/test-suites/dataflow/py37/build.gradle +++ b/sdks/python/test-suites/dataflow/py37/build.gradle @@ -20,9 +20,14 @@ apply plugin: org.apache.beam.gradle.BeamModulePlugin applyPythonNature() enablePythonPerformanceTest() +dependencies { + distTarBall project(path: ":sdks:python", configuration: "distTarBall") +} + // Required to setup a Python 3 virtualenv. pythonVersion = '3.7' + def runScriptsDir = "${project.rootDir}/sdks/python/scripts" // Basic test options for ITs running on Jenkins. @@ -32,7 +37,7 @@ def basicTestOpts = [ "--process-timeout=4500", // timeout of whole command execution ] -task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) { +task preCommitIT(dependsOn: [':sdks:python:sdist', 'installGcpTest']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -51,7 +56,7 @@ task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) { ] def cmdArgs = project.mapToArgString([ "test_opts": testOpts, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "preCommitIT-df-py37" ]) @@ -63,7 +68,7 @@ task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) { } } -task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { +task postCommitIT(dependsOn: [':sdks:python:sdist', 'installGcpTest']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -73,7 +78,7 @@ task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { def cmdArgs = project.mapToArgString([ "test_opts": testOpts, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "postCommitIT-df-py37" ]) @@ -84,7 +89,7 @@ task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) { } } -task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { +task validatesRunnerBatchTests(dependsOn: ['installGcpTest', ':sdks:python:sdist']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -93,7 +98,7 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { def testOpts = basicTestOpts + ["--attr=ValidatesRunner"] def cmdArgs = project.mapToArgString([ "test_opts": testOpts, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "validatesRunnerBatchTests-df-py37" ]) @@ -104,7 +109,7 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { } } -task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { +task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', ':sdks:python:sdist']) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath @@ -114,7 +119,7 @@ task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-streaming"] def argMap = ["test_opts": testOpts, "streaming": "true", - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "validatesRunnerStreamingTests-df-py37"] def cmdArgs = project.mapToArgString(argMap) From 1545120848edcfe65869069f4167a7a68aca2ec2 Mon Sep 17 00:00:00 2001 From: Mark Liu Date: Tue, 6 Aug 2019 14:52:52 -0700 Subject: [PATCH 2/3] Add missing dependency and source copy in tox test --- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index f1f60dad52e3..0c42e7fa8461 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1921,8 +1921,15 @@ class BeamModulePlugin implements Plugin { project.ext.toxTask = { name, tox_env -> project.tasks.create(name) { - dependsOn = [':sdks:python:sdist'] + dependsOn 'setupVirtualenv' + dependsOn ':sdks:python:sdist' + doLast { + // Python source directory is also tox execution workspace, We want + // to isolate them per tox suite to avoid conflict when running + // multiple tox suites in parallel. + project.copy { from project.pythonSdkDeps; into copiedSrcRoot } + def copiedPyRoot = "${copiedSrcRoot}/sdks/python" def distTarBall = "${pythonRootDir}/build/apache-beam.tar.gz" project.exec { From b3c2915e9068080d17a453ab3199654bd3741903 Mon Sep 17 00:00:00 2001 From: Mark Liu Date: Wed, 7 Aug 2019 13:49:34 -0700 Subject: [PATCH 3/3] Build sdk tarball before running installGcpTest task --- .../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 0c42e7fa8461..e212b1e33f3f 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1884,6 +1884,7 @@ class BeamModulePlugin implements Plugin { } } } + project.installGcpTest.mustRunAfter project.configurations.distTarBall project.task('cleanPython') { doLast {