From 6e73cd23f308636d03ecee81d518998cb5b40d8d Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 16 Nov 2022 14:37:06 -0500 Subject: [PATCH 01/23] WIP: Shard python precommit --- .../job_PreCommit_Python_Internal.groovy | 40 +++++++++++++++++++ sdks/python/test-suites/tox/common.gradle | 12 +++--- sdks/python/test-suites/tox/py37/build.gradle | 6 ++- sdks/python/test-suites/tox/py38/build.gradle | 40 ++++++++++--------- 4 files changed, 72 insertions(+), 26 deletions(-) create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Internal.groovy diff --git a/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy b/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy new file mode 100644 index 000000000000..ea5ac358abcb --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python Internal', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/internal/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/sdks/python/test-suites/tox/common.gradle b/sdks/python/test-suites/tox/common.gradle index 61802ac9c45e..6d80617e12f5 100644 --- a/sdks/python/test-suites/tox/common.gradle +++ b/sdks/python/test-suites/tox/common.gradle @@ -18,19 +18,21 @@ def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') -toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}" +def posargs = project.findProperty("posargs") + +toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}", posargs test.dependsOn "testPython${pythonVersionSuffix}" -toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud" +toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud", posargs test.dependsOn "testPy${pythonVersionSuffix}Cloud" -toxTask "testPy${pythonVersionSuffix}Dask", "py${pythonVersionSuffix}-dask" +toxTask "testPy${pythonVersionSuffix}Dask", "py${pythonVersionSuffix}-dask", posargs test.dependsOn "testPy${pythonVersionSuffix}Dask" -toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython" +toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython", posargs test.dependsOn "testPy${pythonVersionSuffix}Cython" -toxTask "testPy38CloudCoverage", "py38-cloudcoverage" +toxTask "testPy38CloudCoverage", "py38-cloudcoverage", posargs test.dependsOn "testPy38CloudCoverage" project.tasks.register("preCommitPy${pythonVersionSuffix}") { diff --git a/sdks/python/test-suites/tox/py37/build.gradle b/sdks/python/test-suites/tox/py37/build.gradle index 2ea0e46ca5be..6dab0b317389 100644 --- a/sdks/python/test-suites/tox/py37/build.gradle +++ b/sdks/python/test-suites/tox/py37/build.gradle @@ -26,13 +26,15 @@ applyPythonNature() // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.7' +def posargs = project.findProperty("posargs") + task lint {} check.dependsOn lint -toxTask "lintPy37", "py37-lint" +toxTask "lintPy37", "py37-lint", posargs lint.dependsOn lintPy37 -toxTask "mypyPy37", "py37-mypy" +toxTask "mypyPy37", "py37-mypy", posargs lint.dependsOn mypyPy37 apply from: "../common.gradle" diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 6c2cdf7f72b9..854f2ab72f96 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -26,6 +26,8 @@ applyPythonNature() // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.8' +def posargs = project.findProperty("posargs") + toxTask "formatter", "py3-yapf-check" check.dependsOn formatter @@ -36,73 +38,73 @@ testPy38Cython.mustRunAfter testPython38, testPy38CloudCoverage // Create a test task for each major version of pyarrow // TODO(BEAM-12000): Move these to Py 3.9. -toxTask "testPy38pyarrow-0", "py38-pyarrow-0" +toxTask "testPy38pyarrow-0", "py38-pyarrow-0", posargs test.dependsOn "testPy38pyarrow-0" preCommitPy38.dependsOn "testPy38pyarrow-0" -toxTask "testPy38pyarrow-1", "py38-pyarrow-1" +toxTask "testPy38pyarrow-1", "py38-pyarrow-1", posargs test.dependsOn "testPy38pyarrow-1" preCommitPy38.dependsOn "testPy38pyarrow-1" -toxTask "testPy38pyarrow-2", "py38-pyarrow-2" +toxTask "testPy38pyarrow-2", "py38-pyarrow-2", posargs test.dependsOn "testPy38pyarrow-2" preCommitPy38.dependsOn "testPy38pyarrow-2" -toxTask "testPy38pyarrow-3", "py38-pyarrow-3" +toxTask "testPy38pyarrow-3", "py38-pyarrow-3", posargs test.dependsOn "testPy38pyarrow-3" preCommitPy38.dependsOn "testPy38pyarrow-3" -toxTask "testPy38pyarrow-4", "py38-pyarrow-4" +toxTask "testPy38pyarrow-4", "py38-pyarrow-4", posargs test.dependsOn "testPy38pyarrow-4" preCommitPy38.dependsOn "testPy38pyarrow-4" -toxTask "testPy38pyarrow-5", "py38-pyarrow-5" +toxTask "testPy38pyarrow-5", "py38-pyarrow-5", posargs test.dependsOn "testPy38pyarrow-5" preCommitPy38.dependsOn "testPy38pyarrow-5" -toxTask "testPy38pyarrow-6", "py38-pyarrow-6" +toxTask "testPy38pyarrow-6", "py38-pyarrow-6", posargs test.dependsOn "testPy38pyarrow-6" preCommitPy38.dependsOn "testPy38pyarrow-6" -toxTask "testPy38pyarrow-7", "py38-pyarrow-7" +toxTask "testPy38pyarrow-7", "py38-pyarrow-7", posargs test.dependsOn "testPy38pyarrow-7" preCommitPy38.dependsOn "testPy38pyarrow-7" -toxTask "testPy38pyarrow-8", "py38-pyarrow-8" +toxTask "testPy38pyarrow-8", "py38-pyarrow-8", posargs test.dependsOn "testPy38pyarrow-8" preCommitPy38.dependsOn "testPy38pyarrow-8" -toxTask "testPy38pyarrow-9", "py38-pyarrow-9" +toxTask "testPy38pyarrow-9", "py38-pyarrow-9", posargs test.dependsOn "testPy38pyarrow-9" preCommitPy38.dependsOn "testPy38pyarrow-9" // Create a test task for each minor version of pandas -toxTask "testPy38pandas-11", "py38-pandas-11" +toxTask "testPy38pandas-11", "py38-pandas-11", posargs test.dependsOn "testPy38pandas-11" preCommitPy38.dependsOn "testPy38pandas-11" -toxTask "testPy38pandas-12", "py38-pandas-12" +toxTask "testPy38pandas-12", "py38-pandas-12", posargs test.dependsOn "testPy38pandas-12" preCommitPy38.dependsOn "testPy38pandas-12" -toxTask "testPy38pandas-13", "py38-pandas-13" +toxTask "testPy38pandas-13", "py38-pandas-13", posargs test.dependsOn "testPy38pandas-13" preCommitPy38.dependsOn "testPy38pandas-13" -toxTask "testPy38pandas-14", "py38-pandas-14" +toxTask "testPy38pandas-14", "py38-pandas-14", posargs test.dependsOn "testPy38pandas-14" preCommitPy38.dependsOn "testPy38pandas-14" // Create a test task for each minor version of pytorch -toxTask "testPy38pytorch-19", "py38-pytorch-19" +toxTask "testPy38pytorch-19", "py38-pytorch-19", posargs test.dependsOn "testPy38pytorch-19" preCommitPy38.dependsOn "testPy38pytorch-19" -toxTask "testPy38pytorch-110", "py38-pytorch-110" +toxTask "testPy38pytorch-110", "py38-pytorch-110", posargs test.dependsOn "testPy38pytorch-110" preCommitPy38.dependsOn "testPy38pytorch-110" -toxTask "whitespacelint", "whitespacelint" +toxTask "whitespacelint", "whitespacelint", posargs task archiveFilesToLint(type: Zip) { archiveFileName = "files-to-whitespacelint.zip" @@ -124,9 +126,9 @@ whitespacelint.dependsOn archiveFilesToLint, unpackFilesToLint unpackFilesToLint.dependsOn archiveFilesToLint archiveFilesToLint.dependsOn cleanPython -toxTask "jest", "jest" +toxTask "jest", "jest", posargs -toxTask "eslint", "eslint" +toxTask "eslint", "eslint", posargs task copyTsSource(type: Copy) { from ("$rootProject.projectDir") { From 97518ed31c4751b3b401a13cef24c0fbca1f658f Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 16 Nov 2022 14:55:42 -0500 Subject: [PATCH 02/23] Naming --- .test-infra/jenkins/job_PreCommit_Python_Internal.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy b/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy index ea5ac358abcb..3a719a931914 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy @@ -20,7 +20,7 @@ import PrecommitJobBuilder PrecommitJobBuilder builder = new PrecommitJobBuilder( scope: this, - nameBase: 'Python Internal', + nameBase: 'Python_Internal', gradleTask: ':pythonPreCommit', gradleSwitches: [ '-Pposargs=apache_beam/internal/' @@ -28,7 +28,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( timeoutMins: 180, triggerPathPatterns: [ '^model/.*$', - '^sdks/python/.*$', + '^sdks/python/apache_beam/internal/metrics/.*$', // TODO - update this to actual paths '^release/.*$', ] ) From 5e701ae9076c3af61301ec20403c36c54b881b13 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 9 Dec 2022 15:02:54 -0500 Subject: [PATCH 03/23] Formatting --- sdks/python/test-suites/tox/common.gradle | 10 ++--- sdks/python/test-suites/tox/py37/build.gradle | 4 +- sdks/python/test-suites/tox/py38/build.gradle | 38 +++++++++---------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/sdks/python/test-suites/tox/common.gradle b/sdks/python/test-suites/tox/common.gradle index 6d80617e12f5..d6033ba8664a 100644 --- a/sdks/python/test-suites/tox/common.gradle +++ b/sdks/python/test-suites/tox/common.gradle @@ -20,19 +20,19 @@ def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') def posargs = project.findProperty("posargs") -toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}", posargs +toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}", "${posargs}" test.dependsOn "testPython${pythonVersionSuffix}" -toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud", posargs +toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud", "${posargs}" test.dependsOn "testPy${pythonVersionSuffix}Cloud" -toxTask "testPy${pythonVersionSuffix}Dask", "py${pythonVersionSuffix}-dask", posargs +toxTask "testPy${pythonVersionSuffix}Dask", "py${pythonVersionSuffix}-dask", "${posargs}" test.dependsOn "testPy${pythonVersionSuffix}Dask" -toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython", posargs +toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython", "${posargs}" test.dependsOn "testPy${pythonVersionSuffix}Cython" -toxTask "testPy38CloudCoverage", "py38-cloudcoverage", posargs +toxTask "testPy38CloudCoverage", "py38-cloudcoverage", "${posargs}" test.dependsOn "testPy38CloudCoverage" project.tasks.register("preCommitPy${pythonVersionSuffix}") { diff --git a/sdks/python/test-suites/tox/py37/build.gradle b/sdks/python/test-suites/tox/py37/build.gradle index 6dab0b317389..94abd84d35bd 100644 --- a/sdks/python/test-suites/tox/py37/build.gradle +++ b/sdks/python/test-suites/tox/py37/build.gradle @@ -31,10 +31,10 @@ def posargs = project.findProperty("posargs") task lint {} check.dependsOn lint -toxTask "lintPy37", "py37-lint", posargs +toxTask "lintPy37", "py37-lint", "${posargs}" lint.dependsOn lintPy37 -toxTask "mypyPy37", "py37-mypy", posargs +toxTask "mypyPy37", "py37-mypy", "${posargs}" lint.dependsOn mypyPy37 apply from: "../common.gradle" diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 854f2ab72f96..f22f5aa6a52c 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -38,73 +38,73 @@ testPy38Cython.mustRunAfter testPython38, testPy38CloudCoverage // Create a test task for each major version of pyarrow // TODO(BEAM-12000): Move these to Py 3.9. -toxTask "testPy38pyarrow-0", "py38-pyarrow-0", posargs +toxTask "testPy38pyarrow-0", "py38-pyarrow-0", "${posargs}" test.dependsOn "testPy38pyarrow-0" preCommitPy38.dependsOn "testPy38pyarrow-0" -toxTask "testPy38pyarrow-1", "py38-pyarrow-1", posargs +toxTask "testPy38pyarrow-1", "py38-pyarrow-1", "${posargs}" test.dependsOn "testPy38pyarrow-1" preCommitPy38.dependsOn "testPy38pyarrow-1" -toxTask "testPy38pyarrow-2", "py38-pyarrow-2", posargs +toxTask "testPy38pyarrow-2", "py38-pyarrow-2", "${posargs}" test.dependsOn "testPy38pyarrow-2" preCommitPy38.dependsOn "testPy38pyarrow-2" -toxTask "testPy38pyarrow-3", "py38-pyarrow-3", posargs +toxTask "testPy38pyarrow-3", "py38-pyarrow-3", "${posargs}" test.dependsOn "testPy38pyarrow-3" preCommitPy38.dependsOn "testPy38pyarrow-3" -toxTask "testPy38pyarrow-4", "py38-pyarrow-4", posargs +toxTask "testPy38pyarrow-4", "py38-pyarrow-4", "${posargs}" test.dependsOn "testPy38pyarrow-4" preCommitPy38.dependsOn "testPy38pyarrow-4" -toxTask "testPy38pyarrow-5", "py38-pyarrow-5", posargs +toxTask "testPy38pyarrow-5", "py38-pyarrow-5", "${posargs}" test.dependsOn "testPy38pyarrow-5" preCommitPy38.dependsOn "testPy38pyarrow-5" -toxTask "testPy38pyarrow-6", "py38-pyarrow-6", posargs +toxTask "testPy38pyarrow-6", "py38-pyarrow-6", "${posargs}" test.dependsOn "testPy38pyarrow-6" preCommitPy38.dependsOn "testPy38pyarrow-6" -toxTask "testPy38pyarrow-7", "py38-pyarrow-7", posargs +toxTask "testPy38pyarrow-7", "py38-pyarrow-7", "${posargs}" test.dependsOn "testPy38pyarrow-7" preCommitPy38.dependsOn "testPy38pyarrow-7" -toxTask "testPy38pyarrow-8", "py38-pyarrow-8", posargs +toxTask "testPy38pyarrow-8", "py38-pyarrow-8", "${posargs}" test.dependsOn "testPy38pyarrow-8" preCommitPy38.dependsOn "testPy38pyarrow-8" -toxTask "testPy38pyarrow-9", "py38-pyarrow-9", posargs +toxTask "testPy38pyarrow-9", "py38-pyarrow-9", "${posargs}" test.dependsOn "testPy38pyarrow-9" preCommitPy38.dependsOn "testPy38pyarrow-9" // Create a test task for each minor version of pandas -toxTask "testPy38pandas-11", "py38-pandas-11", posargs +toxTask "testPy38pandas-11", "py38-pandas-11", "${posargs}" test.dependsOn "testPy38pandas-11" preCommitPy38.dependsOn "testPy38pandas-11" -toxTask "testPy38pandas-12", "py38-pandas-12", posargs +toxTask "testPy38pandas-12", "py38-pandas-12", "${posargs}" test.dependsOn "testPy38pandas-12" preCommitPy38.dependsOn "testPy38pandas-12" -toxTask "testPy38pandas-13", "py38-pandas-13", posargs +toxTask "testPy38pandas-13", "py38-pandas-13", "${posargs}" test.dependsOn "testPy38pandas-13" preCommitPy38.dependsOn "testPy38pandas-13" -toxTask "testPy38pandas-14", "py38-pandas-14", posargs +toxTask "testPy38pandas-14", "py38-pandas-14", "${posargs}" test.dependsOn "testPy38pandas-14" preCommitPy38.dependsOn "testPy38pandas-14" // Create a test task for each minor version of pytorch -toxTask "testPy38pytorch-19", "py38-pytorch-19", posargs +toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}" test.dependsOn "testPy38pytorch-19" preCommitPy38.dependsOn "testPy38pytorch-19" -toxTask "testPy38pytorch-110", "py38-pytorch-110", posargs +toxTask "testPy38pytorch-110", "py38-pytorch-110", "${posargs}" test.dependsOn "testPy38pytorch-110" preCommitPy38.dependsOn "testPy38pytorch-110" -toxTask "whitespacelint", "whitespacelint", posargs +toxTask "whitespacelint", "whitespacelint", "${posargs}" task archiveFilesToLint(type: Zip) { archiveFileName = "files-to-whitespacelint.zip" @@ -126,9 +126,9 @@ whitespacelint.dependsOn archiveFilesToLint, unpackFilesToLint unpackFilesToLint.dependsOn archiveFilesToLint archiveFilesToLint.dependsOn cleanPython -toxTask "jest", "jest", posargs +toxTask "jest", "jest", "${posargs}" -toxTask "eslint", "eslint", posargs +toxTask "eslint", "eslint", "${posargs}" task copyTsSource(type: Copy) { from ("$rootProject.projectDir") { From 906b8fbd0b921971c82a74ed0628c9636505cbab Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 12 Dec 2022 12:46:17 -0500 Subject: [PATCH 04/23] Allow empty test suites --- sdks/python/scripts/run_pytest.sh | 7 +------ sdks/python/tox.ini | 8 +++++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/sdks/python/scripts/run_pytest.sh b/sdks/python/scripts/run_pytest.sh index 6c7d5c46e78e..3df342aaf912 100755 --- a/sdks/python/scripts/run_pytest.sh +++ b/sdks/python/scripts/run_pytest.sh @@ -43,12 +43,7 @@ pytest -o junit_suite_name=${envname}_no_xdist \ --junitxml=pytest_${envname}_no_xdist.xml -m 'no_xdist' ${pytest_args} --pyargs ${posargs} status2=$? -# Exit with error if no tests were run in either suite (status code 5). -if [[ $status1 == 5 && $status2 == 5 ]]; then - exit $status1 -fi - -# Exit with error if one of the statuses has an error that's not 5. +# Exit with error if one of the statuses has an error that's not 5 (no tests run). if [[ $status1 != 0 && $status1 != 5 ]]; then exit $status1 fi diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 33ec39c41892..c92fb0e02e75 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -259,8 +259,9 @@ commands = # Log pyarrow and numpy version for debugging /bin/sh -c "pip freeze | grep -E '(pyarrow|numpy)'" # Run pytest directly rather using run_pytest.sh. It doesn't handle - # selecting tests with -m (BEAM-12985) - pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs} + # selecting tests with -m (BEAM-12985). + # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. + /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' [testenv:py{37,38,39,310}-pandas-{11,12,13,14}] deps = @@ -284,4 +285,5 @@ commands = # Log torch version for debugging /bin/sh -c "pip freeze | grep -E torch" # Run all PyTorch unit tests - pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs} + # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. + /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' From 057428f368f9b0d383e319f7c138974523dcbcce Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 12 Dec 2022 14:28:10 -0500 Subject: [PATCH 05/23] Fix no posargs case --- sdks/python/test-suites/tox/common.gradle | 2 +- sdks/python/test-suites/tox/py37/build.gradle | 2 +- sdks/python/test-suites/tox/py38/build.gradle | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/test-suites/tox/common.gradle b/sdks/python/test-suites/tox/common.gradle index d6033ba8664a..5f5a1012f563 100644 --- a/sdks/python/test-suites/tox/common.gradle +++ b/sdks/python/test-suites/tox/common.gradle @@ -18,7 +18,7 @@ def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') -def posargs = project.findProperty("posargs") +def posargs = project.findProperty("posargs") ?: "" toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}", "${posargs}" test.dependsOn "testPython${pythonVersionSuffix}" diff --git a/sdks/python/test-suites/tox/py37/build.gradle b/sdks/python/test-suites/tox/py37/build.gradle index 94abd84d35bd..744ca6750629 100644 --- a/sdks/python/test-suites/tox/py37/build.gradle +++ b/sdks/python/test-suites/tox/py37/build.gradle @@ -26,7 +26,7 @@ applyPythonNature() // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.7' -def posargs = project.findProperty("posargs") +def posargs = project.findProperty("posargs") ?: "" task lint {} check.dependsOn lint diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index f22f5aa6a52c..71dd59650aa6 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -26,7 +26,7 @@ applyPythonNature() // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.8' -def posargs = project.findProperty("posargs") +def posargs = project.findProperty("posargs") ?: "" toxTask "formatter", "py3-yapf-check" check.dependsOn formatter From d4f008c81b04bf11315cfd7340c58bf73312e3da Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 12 Dec 2022 16:06:38 -0500 Subject: [PATCH 06/23] Fix paths --- .test-infra/jenkins/job_PreCommit_Python.groovy | 3 +++ .test-infra/jenkins/job_PreCommit_Python_Internal.groovy | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index 0b5270f96eef..0f1aac82fbc8 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -22,6 +22,9 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( scope: this, nameBase: 'Python', gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=--ignore=apache_beam/internal/' + ], timeoutMins: 180, triggerPathPatterns: [ '^model/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy b/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy index 3a719a931914..aefae85f6f01 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy @@ -28,7 +28,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( timeoutMins: 180, triggerPathPatterns: [ '^model/.*$', - '^sdks/python/apache_beam/internal/metrics/.*$', // TODO - update this to actual paths + '^sdks/python/apache_beam/internal/.*$', '^release/.*$', ] ) From 285672f01276b594742fa49792e219227359b759 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 12 Dec 2022 16:22:52 -0500 Subject: [PATCH 07/23] Split out integration tests --- .../jenkins/job_PreCommit_PythonIT.groovy | 37 +++++++++++++++++++ build.gradle.kts | 3 ++ 2 files changed, 40 insertions(+) create mode 100644 .test-infra/jenkins/job_PreCommit_PythonIT.groovy diff --git a/.test-infra/jenkins/job_PreCommit_PythonIT.groovy b/.test-infra/jenkins/job_PreCommit_PythonIT.groovy new file mode 100644 index 000000000000..2ef31a3937c3 --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_PythonIT.groovy @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python Integration', + gradleTask: ':pythonPreCommitIT', + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/build.gradle.kts b/build.gradle.kts index 2556db7bc9ae..66322e81f918 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -311,6 +311,9 @@ tasks.register("pythonPreCommit") { dependsOn(":sdks:python:test-suites:tox:py38:preCommitPy38") dependsOn(":sdks:python:test-suites:tox:py39:preCommitPy39") dependsOn(":sdks:python:test-suites:tox:py310:preCommitPy310") +} + +tasks.register("pythonPreCommitIT") { dependsOn(":sdks:python:test-suites:dataflow:preCommitIT") dependsOn(":sdks:python:test-suites:dataflow:preCommitIT_V2") } From 6b1f4819b6fce5a22db5122f988679e1e6012744 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Tue, 13 Dec 2022 09:30:33 -0500 Subject: [PATCH 08/23] Do common tasks in IT suite --- build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle.kts b/build.gradle.kts index 66322e81f918..b0bf9a75e680 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -314,6 +314,7 @@ tasks.register("pythonPreCommit") { } tasks.register("pythonPreCommitIT") { + dependsOn(":sdks:python:test-suites:tox:pycommon:preCommitPyCommon") dependsOn(":sdks:python:test-suites:dataflow:preCommitIT") dependsOn(":sdks:python:test-suites:dataflow:preCommitIT_V2") } From 5a5023f16046a9d35fc52ff55c5ecdc7a3f8893c Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Tue, 13 Dec 2022 09:35:42 -0500 Subject: [PATCH 09/23] Fix name base --- .test-infra/jenkins/job_PreCommit_PythonIT.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PreCommit_PythonIT.groovy b/.test-infra/jenkins/job_PreCommit_PythonIT.groovy index 2ef31a3937c3..3d563748d0b5 100644 --- a/.test-infra/jenkins/job_PreCommit_PythonIT.groovy +++ b/.test-infra/jenkins/job_PreCommit_PythonIT.groovy @@ -20,7 +20,7 @@ import PrecommitJobBuilder PrecommitJobBuilder builder = new PrecommitJobBuilder( scope: this, - nameBase: 'Python Integration', + nameBase: 'Python_Integration', gradleTask: ':pythonPreCommitIT', timeoutMins: 180, triggerPathPatterns: [ From 19041fb3c66044afeeef32ca01c7b70c76e24390 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Tue, 13 Dec 2022 13:27:52 -0500 Subject: [PATCH 10/23] More splitting --- .../jenkins/job_PreCommit_Python.groovy | 2 +- .../job_PreCommit_Python_Coders.groovy | 45 +++++++++++++++++++ .../job_PreCommit_Python_Dataframe.groovy | 40 +++++++++++++++++ .../job_PreCommit_Python_Examples.groovy | 40 +++++++++++++++++ .../job_PreCommit_Python_Internal.groovy | 3 ++ 5 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Coders.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Dataframe.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Examples.groovy diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index 0f1aac82fbc8..7878f85d4d38 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -23,7 +23,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=--ignore=apache_beam/internal/' + '-Pposargs=\'--ignore=apache_beam/coders/ --ignore=apache_beam/dataframe/ --ignore=apache_beam/examples/ --ignore=apache_beam/internal/\'' ], timeoutMins: 180, triggerPathPatterns: [ diff --git a/.test-infra/jenkins/job_PreCommit_Python_Coders.groovy b/.test-infra/jenkins/job_PreCommit_Python_Coders.groovy new file mode 100644 index 000000000000..b8f68e7823b3 --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Coders.groovy @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Coders', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/coders/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/apache_beam/coders/.*$', + '^sdks/python/apache_beam/internal/.*$', + '^sdks/python/apache_beam/portability/.*$', + '^sdks/python/apache_beam/transforms/.*$', + '^sdks/python/apache_beam/typehints/.*$', + '^sdks/python/apache_beam/utils/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Dataframe.groovy b/.test-infra/jenkins/job_PreCommit_Python_Dataframe.groovy new file mode 100644 index 000000000000..986faa717885 --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Dataframe.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Dataframe', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/dataframe/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Examples.groovy b/.test-infra/jenkins/job_PreCommit_Python_Examples.groovy new file mode 100644 index 000000000000..12e43e155fab --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Examples.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Examples', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/examples/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy b/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy index aefae85f6f01..5319467a3f69 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy @@ -29,6 +29,9 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( triggerPathPatterns: [ '^model/.*$', '^sdks/python/apache_beam/internal/.*$', + '^sdks/python/apache_beam/metrics/.*$', + '^sdks/python/apache_beam/options/.*$', + '^sdks/python/apache_beam/utils/.*$', '^release/.*$', ] ) From 9fc21e1cf1169f81170de8ee49584b0e19c07c7c Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Tue, 13 Dec 2022 13:56:31 -0500 Subject: [PATCH 11/23] Fix quotes --- .test-infra/jenkins/job_PreCommit_Python.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index 7878f85d4d38..7f9115733816 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -23,7 +23,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=\'--ignore=apache_beam/coders/ --ignore=apache_beam/dataframe/ --ignore=apache_beam/examples/ --ignore=apache_beam/internal/\'' + '-Pposargs="--ignore=apache_beam/coders/ --ignore=apache_beam/dataframe/ --ignore=apache_beam/examples/ --ignore=apache_beam/internal/"' ], timeoutMins: 180, triggerPathPatterns: [ From c90a0418d1ec7af5abf539ee7d78d123c3844a6c Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Tue, 13 Dec 2022 14:12:38 -0500 Subject: [PATCH 12/23] try ignore-glob --- .test-infra/jenkins/job_PreCommit_Python.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index 7f9115733816..7fa600a7be82 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -23,7 +23,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs="--ignore=apache_beam/coders/ --ignore=apache_beam/dataframe/ --ignore=apache_beam/examples/ --ignore=apache_beam/internal/"' + '-Pposargs=--ignore-glob=apache_beam/{coders,dataframe,examples,internal}/{**/*,*}' // Exclude any tests that are covered by other suites ], timeoutMins: 180, triggerPathPatterns: [ From 566e24e7c32229229ec115dd2fae5c108e2c9432 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 15 Dec 2022 13:59:02 -0500 Subject: [PATCH 13/23] Add groovy files per subfolder --- .../jenkins/job_PreCommit_Python.groovy | 2 +- .../jenkins/job_PreCommit_Python_ML.groovy | 40 +++++++++++++++++++ .../job_PreCommit_Python_Options.groovy | 40 +++++++++++++++++++ .../job_PreCommit_Python_Portability.groovy | 40 +++++++++++++++++++ .../job_PreCommit_Python_Runners.groovy | 40 +++++++++++++++++++ .../job_PreCommit_Python_Testing.groovy | 40 +++++++++++++++++++ .../jenkins/job_PreCommit_Python_Tools.groovy | 40 +++++++++++++++++++ .../job_PreCommit_Python_Transforms.groovy | 40 +++++++++++++++++++ .../job_PreCommit_Python_Typehints.groovy | 40 +++++++++++++++++++ .../jenkins/job_PreCommit_Python_Utils.groovy | 40 +++++++++++++++++++ 10 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 .test-infra/jenkins/job_PreCommit_Python_ML.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Options.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Portability.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Runners.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Testing.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Tools.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Transforms.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Typehints.groovy create mode 100644 .test-infra/jenkins/job_PreCommit_Python_Utils.groovy diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index 7fa600a7be82..6d9f29c3a47c 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -23,7 +23,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=--ignore-glob=apache_beam/{coders,dataframe,examples,internal}/{**/*,*}' // Exclude any tests that are covered by other suites + '-Pposargs=--ignore-glob=apache_beam/{coders,dataframe,examples,internal,ml,options,portability,runners,testing,tools,transforms,typehints,utils}/{**/*,*}' // Exclude any tests that are covered by other suites ], timeoutMins: 180, triggerPathPatterns: [ diff --git a/.test-infra/jenkins/job_PreCommit_Python_ML.groovy b/.test-infra/jenkins/job_PreCommit_Python_ML.groovy new file mode 100644 index 000000000000..ec86182215ca --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_ML.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Examples', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/ml/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Options.groovy b/.test-infra/jenkins/job_PreCommit_Python_Options.groovy new file mode 100644 index 000000000000..d900ea3cbcce --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Options.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Options', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/options/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Portability.groovy b/.test-infra/jenkins/job_PreCommit_Python_Portability.groovy new file mode 100644 index 000000000000..279bfd3f5f2a --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Portability.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Portability', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/portability/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Runners.groovy b/.test-infra/jenkins/job_PreCommit_Python_Runners.groovy new file mode 100644 index 000000000000..a9ca1da0d2ea --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Runners.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Runners', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/runners/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Testing.groovy b/.test-infra/jenkins/job_PreCommit_Python_Testing.groovy new file mode 100644 index 000000000000..298e23336289 --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Testing.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Testing', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/testing/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Tools.groovy b/.test-infra/jenkins/job_PreCommit_Python_Tools.groovy new file mode 100644 index 000000000000..01a9eef037e1 --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Tools.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Tools', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/tools/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Transforms.groovy b/.test-infra/jenkins/job_PreCommit_Python_Transforms.groovy new file mode 100644 index 000000000000..1264fffa07b6 --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Transforms.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Transforms', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/transforms/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Typehints.groovy b/.test-infra/jenkins/job_PreCommit_Python_Typehints.groovy new file mode 100644 index 000000000000..9537e8d1ad5e --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Typehints.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Typehints', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/typehints/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Utils.groovy b/.test-infra/jenkins/job_PreCommit_Python_Utils.groovy new file mode 100644 index 000000000000..02ca83c7ac81 --- /dev/null +++ b/.test-infra/jenkins/job_PreCommit_Python_Utils.groovy @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PrecommitJobBuilder + +PrecommitJobBuilder builder = new PrecommitJobBuilder( + scope: this, + nameBase: 'Python_Utils', + gradleTask: ':pythonPreCommit', + gradleSwitches: [ + '-Pposargs=apache_beam/utils/' + ], + timeoutMins: 180, + triggerPathPatterns: [ + '^model/.*$', + '^sdks/python/.*$', + '^release/.*$', + ] + ) +builder.build { + // Publish all test results to Jenkins. + publishers { + archiveJunit('**/pytest*.xml') + } +} From a489ac18fc59f5402f45985a1ef6e58041031e3c Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 16 Dec 2022 10:36:31 -0500 Subject: [PATCH 14/23] name typo --- .test-infra/jenkins/job_PreCommit_Python_ML.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PreCommit_Python_ML.groovy b/.test-infra/jenkins/job_PreCommit_Python_ML.groovy index ec86182215ca..99543f382a3c 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_ML.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_ML.groovy @@ -20,7 +20,7 @@ import PrecommitJobBuilder PrecommitJobBuilder builder = new PrecommitJobBuilder( scope: this, - nameBase: 'Python_Examples', + nameBase: 'Python_ML', gradleTask: ':pythonPreCommit', gradleSwitches: [ '-Pposargs=apache_beam/ml/' From 9949840a75b695271893a0c79c8413a398d14973 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 19 Dec 2022 15:34:20 -0500 Subject: [PATCH 15/23] Filter dataframes tests --- sdks/python/test-suites/tox/py38/build.gradle | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 71dd59650aa6..a4d6ca17b152 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -27,6 +27,7 @@ applyPythonNature() pythonVersion = '3.8' def posargs = project.findProperty("posargs") ?: "" +def runDataframesTests = (posargs == "" || posargs.contains("dataframes")) toxTask "formatter", "py3-yapf-check" check.dependsOn formatter @@ -78,22 +79,25 @@ toxTask "testPy38pyarrow-9", "py38-pyarrow-9", "${posargs}" test.dependsOn "testPy38pyarrow-9" preCommitPy38.dependsOn "testPy38pyarrow-9" -// Create a test task for each minor version of pandas -toxTask "testPy38pandas-11", "py38-pandas-11", "${posargs}" -test.dependsOn "testPy38pandas-11" -preCommitPy38.dependsOn "testPy38pandas-11" +// Dataframes tests won't get auto-filtered by path since they explicitly only run on the dataframes directory in tox.ini +if (runDataframesTests) { + // Create a test task for each minor version of pandas + toxTask "testPy38pandas-11", "py38-pandas-11", "${posargs}" + test.dependsOn "testPy38pandas-11" + preCommitPy38.dependsOn "testPy38pandas-11" -toxTask "testPy38pandas-12", "py38-pandas-12", "${posargs}" -test.dependsOn "testPy38pandas-12" -preCommitPy38.dependsOn "testPy38pandas-12" + toxTask "testPy38pandas-12", "py38-pandas-12", "${posargs}" + test.dependsOn "testPy38pandas-12" + preCommitPy38.dependsOn "testPy38pandas-12" -toxTask "testPy38pandas-13", "py38-pandas-13", "${posargs}" -test.dependsOn "testPy38pandas-13" -preCommitPy38.dependsOn "testPy38pandas-13" + toxTask "testPy38pandas-13", "py38-pandas-13", "${posargs}" + test.dependsOn "testPy38pandas-13" + preCommitPy38.dependsOn "testPy38pandas-13" -toxTask "testPy38pandas-14", "py38-pandas-14", "${posargs}" -test.dependsOn "testPy38pandas-14" -preCommitPy38.dependsOn "testPy38pandas-14" + toxTask "testPy38pandas-14", "py38-pandas-14", "${posargs}" + test.dependsOn "testPy38pandas-14" + preCommitPy38.dependsOn "testPy38pandas-14" +} // Create a test task for each minor version of pytorch toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}" From 08a9da603692703ae938a87588d8575a714cad32 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 22 Dec 2022 16:25:00 -0500 Subject: [PATCH 16/23] Logging --- sdks/python/test-suites/tox/py38/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index a4d6ca17b152..e14b8c896d7d 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -28,6 +28,7 @@ pythonVersion = '3.8' def posargs = project.findProperty("posargs") ?: "" def runDataframesTests = (posargs == "" || posargs.contains("dataframes")) +logger.info('Running dataframes tests: ' + runDataframesTests) toxTask "formatter", "py3-yapf-check" check.dependsOn formatter From 776ca0d29ec887f0e2c2bc15a42d4964c11825da Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 22 Dec 2022 17:00:05 -0500 Subject: [PATCH 17/23] Switch to correct directory syntax --- .test-infra/jenkins/job_PreCommit_Python.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index 6d9f29c3a47c..d01a6c15ecc4 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -23,7 +23,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=--ignore-glob=apache_beam/{coders,dataframe,examples,internal,ml,options,portability,runners,testing,tools,transforms,typehints,utils}/{**/*,*}' // Exclude any tests that are covered by other suites + '-Pposargs=./*.py' // All other tests are covered by different jobs. ], timeoutMins: 180, triggerPathPatterns: [ From 0aff38eb3db63a009d406e643e9af12632ce82ef Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Tue, 27 Dec 2022 10:05:04 -0500 Subject: [PATCH 18/23] Merge master + fix paths --- .test-infra/jenkins/job_PreCommit_Python.groovy | 2 +- sdks/python/test-suites/tox/py38/build.gradle | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index d01a6c15ecc4..25926dc96f3e 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -23,7 +23,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=./*.py' // All other tests are covered by different jobs. + '-Pposargs=apache_beam/*.py' // All other tests are covered by different jobs. ], timeoutMins: 180, triggerPathPatterns: [ diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 3ddc6055b3bd..91d06cddf2e9 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -98,11 +98,11 @@ if (runDataframesTests) { toxTask "testPy38pandas-14", "py38-pandas-14", "${posargs}" test.dependsOn "testPy38pandas-14" preCommitPy38.dependsOn "testPy38pandas-14" -} -toxTask "testPy38pandas-15", "py38-pandas-15" -test.dependsOn "testPy38pandas-15" -preCommitPy38.dependsOn "testPy38pandas-15" + toxTask "testPy38pandas-15", "py38-pandas-15" + test.dependsOn "testPy38pandas-15" + preCommitPy38.dependsOn "testPy38pandas-15" +} // Create a test task for each minor version of pytorch toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}" From 03238222d588f5d3745bfc700d26bd37ed728c80 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Tue, 27 Dec 2022 10:57:08 -0500 Subject: [PATCH 19/23] Consolidate back to reasonable set --- .../jenkins/job_PreCommit_Python.groovy | 2 +- .../job_PreCommit_Python_Coders.groovy | 45 ------------------- .../job_PreCommit_Python_Internal.groovy | 43 ------------------ .../jenkins/job_PreCommit_Python_ML.groovy | 40 ----------------- .../job_PreCommit_Python_Options.groovy | 40 ----------------- .../job_PreCommit_Python_Portability.groovy | 40 ----------------- .../job_PreCommit_Python_Testing.groovy | 40 ----------------- .../jenkins/job_PreCommit_Python_Tools.groovy | 40 ----------------- .../job_PreCommit_Python_Typehints.groovy | 40 ----------------- .../jenkins/job_PreCommit_Python_Utils.groovy | 40 ----------------- 10 files changed, 1 insertion(+), 369 deletions(-) delete mode 100644 .test-infra/jenkins/job_PreCommit_Python_Coders.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Python_Internal.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Python_ML.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Python_Options.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Python_Portability.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Python_Testing.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Python_Tools.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Python_Typehints.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Python_Utils.groovy diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index 25926dc96f3e..3fed71cc6fea 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -23,7 +23,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=apache_beam/*.py' // All other tests are covered by different jobs. + '-Pposargs=\'apache_beam/*.py apache_beam/coders apache_beam/internal apache_beam/ml apache_beam/options apache_beam/portability apache_beam/testing apache_beam/tools apache_beam/typehints apache_beam/utils\'' // All other tests are covered by different jobs. ], timeoutMins: 180, triggerPathPatterns: [ diff --git a/.test-infra/jenkins/job_PreCommit_Python_Coders.groovy b/.test-infra/jenkins/job_PreCommit_Python_Coders.groovy deleted file mode 100644 index b8f68e7823b3..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Python_Coders.groovy +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Python_Coders', - gradleTask: ':pythonPreCommit', - gradleSwitches: [ - '-Pposargs=apache_beam/coders/' - ], - timeoutMins: 180, - triggerPathPatterns: [ - '^model/.*$', - '^sdks/python/apache_beam/coders/.*$', - '^sdks/python/apache_beam/internal/.*$', - '^sdks/python/apache_beam/portability/.*$', - '^sdks/python/apache_beam/transforms/.*$', - '^sdks/python/apache_beam/typehints/.*$', - '^sdks/python/apache_beam/utils/.*$', - '^release/.*$', - ] - ) -builder.build { - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy b/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy deleted file mode 100644 index 5319467a3f69..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Python_Internal.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Python_Internal', - gradleTask: ':pythonPreCommit', - gradleSwitches: [ - '-Pposargs=apache_beam/internal/' - ], - timeoutMins: 180, - triggerPathPatterns: [ - '^model/.*$', - '^sdks/python/apache_beam/internal/.*$', - '^sdks/python/apache_beam/metrics/.*$', - '^sdks/python/apache_beam/options/.*$', - '^sdks/python/apache_beam/utils/.*$', - '^release/.*$', - ] - ) -builder.build { - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Python_ML.groovy b/.test-infra/jenkins/job_PreCommit_Python_ML.groovy deleted file mode 100644 index 99543f382a3c..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Python_ML.groovy +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Python_ML', - gradleTask: ':pythonPreCommit', - gradleSwitches: [ - '-Pposargs=apache_beam/ml/' - ], - timeoutMins: 180, - triggerPathPatterns: [ - '^model/.*$', - '^sdks/python/.*$', - '^release/.*$', - ] - ) -builder.build { - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Options.groovy b/.test-infra/jenkins/job_PreCommit_Python_Options.groovy deleted file mode 100644 index d900ea3cbcce..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Python_Options.groovy +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Python_Options', - gradleTask: ':pythonPreCommit', - gradleSwitches: [ - '-Pposargs=apache_beam/options/' - ], - timeoutMins: 180, - triggerPathPatterns: [ - '^model/.*$', - '^sdks/python/.*$', - '^release/.*$', - ] - ) -builder.build { - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Portability.groovy b/.test-infra/jenkins/job_PreCommit_Python_Portability.groovy deleted file mode 100644 index 279bfd3f5f2a..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Python_Portability.groovy +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Python_Portability', - gradleTask: ':pythonPreCommit', - gradleSwitches: [ - '-Pposargs=apache_beam/portability/' - ], - timeoutMins: 180, - triggerPathPatterns: [ - '^model/.*$', - '^sdks/python/.*$', - '^release/.*$', - ] - ) -builder.build { - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Testing.groovy b/.test-infra/jenkins/job_PreCommit_Python_Testing.groovy deleted file mode 100644 index 298e23336289..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Python_Testing.groovy +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Python_Testing', - gradleTask: ':pythonPreCommit', - gradleSwitches: [ - '-Pposargs=apache_beam/testing/' - ], - timeoutMins: 180, - triggerPathPatterns: [ - '^model/.*$', - '^sdks/python/.*$', - '^release/.*$', - ] - ) -builder.build { - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Tools.groovy b/.test-infra/jenkins/job_PreCommit_Python_Tools.groovy deleted file mode 100644 index 01a9eef037e1..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Python_Tools.groovy +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Python_Tools', - gradleTask: ':pythonPreCommit', - gradleSwitches: [ - '-Pposargs=apache_beam/tools/' - ], - timeoutMins: 180, - triggerPathPatterns: [ - '^model/.*$', - '^sdks/python/.*$', - '^release/.*$', - ] - ) -builder.build { - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Typehints.groovy b/.test-infra/jenkins/job_PreCommit_Python_Typehints.groovy deleted file mode 100644 index 9537e8d1ad5e..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Python_Typehints.groovy +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Python_Typehints', - gradleTask: ':pythonPreCommit', - gradleSwitches: [ - '-Pposargs=apache_beam/typehints/' - ], - timeoutMins: 180, - triggerPathPatterns: [ - '^model/.*$', - '^sdks/python/.*$', - '^release/.*$', - ] - ) -builder.build { - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Python_Utils.groovy b/.test-infra/jenkins/job_PreCommit_Python_Utils.groovy deleted file mode 100644 index 02ca83c7ac81..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Python_Utils.groovy +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Python_Utils', - gradleTask: ':pythonPreCommit', - gradleSwitches: [ - '-Pposargs=apache_beam/utils/' - ], - timeoutMins: 180, - triggerPathPatterns: [ - '^model/.*$', - '^sdks/python/.*$', - '^release/.*$', - ] - ) -builder.build { - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } -} From d18f2264d9f5dbff6ed7f12306e4908817482341 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Tue, 27 Dec 2022 11:34:52 -0500 Subject: [PATCH 20/23] Add jobs to README --- .test-infra/jenkins/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index e53dae86c458..8bc4b6f9e257 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -44,6 +44,11 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PreCommit_Portable_Python | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Phrase/) | `Run Portable_Python PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Cron) | | beam_PreCommit_PythonLint | [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Phrase/) | `Run PythonLint PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Cron) | | beam_PreCommit_Python | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Phrase/) | `Run Python PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron) | +| beam_PreCommit_Python_Integration | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Integration_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Integration_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Integration_Phrase/) | `Run Python_Integration PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Integration_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Integration_Cron) | +| beam_PreCommit_Python_Dataframe | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Dataframe_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Dataframe_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Dataframe_Phrase/) | `Run Python_TODO PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Dataframe_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Dataframe_Cron) | +| beam_PreCommit_Python_Examples | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Phrase/) | `Run Python_TODO PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Cron) | +| beam_PreCommit_Python_Runners | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Phrase/) | `Run Python_TODO PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Cron) | +| beam_PreCommit_Python_Transforms | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Phrase/) | `Run Python_TODO PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Cron) | | beam_PreCommit_PythonDocker | [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Phrase/) | `Run PythonDocker PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Cron/) | | beam_PreCommit_PythonDocs| [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Phrase/) | `Run PythonDocs PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/) | | beam_PreCommit_Python_PVR_Flink | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Phrase/) | `Run Python_PVR_Flink PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron) | From bea8f4a0dea2b64553c4bfcf6941c26af9478bcb Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Tue, 27 Dec 2022 12:05:06 -0500 Subject: [PATCH 21/23] Try full quotes --- .test-infra/jenkins/job_PreCommit_Python.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index 3fed71cc6fea..6728ce71cf0b 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -23,7 +23,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=\'apache_beam/*.py apache_beam/coders apache_beam/internal apache_beam/ml apache_beam/options apache_beam/portability apache_beam/testing apache_beam/tools apache_beam/typehints apache_beam/utils\'' // All other tests are covered by different jobs. + '-Pposargs=\"apache_beam/*.py apache_beam/coders apache_beam/internal apache_beam/ml apache_beam/options apache_beam/portability apache_beam/testing apache_beam/tools apache_beam/typehints apache_beam/utils\"' // All other tests are covered by different jobs. ], timeoutMins: 180, triggerPathPatterns: [ From 0508e933940f4f02c684518d62de102001854d20 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 28 Dec 2022 11:14:37 -0500 Subject: [PATCH 22/23] Add in missing tests --- .test-infra/jenkins/job_PreCommit_Python.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PreCommit_Python.groovy b/.test-infra/jenkins/job_PreCommit_Python.groovy index 6728ce71cf0b..289a7c846b3a 100644 --- a/.test-infra/jenkins/job_PreCommit_Python.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python.groovy @@ -23,7 +23,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Python', gradleTask: ':pythonPreCommit', gradleSwitches: [ - '-Pposargs=\"apache_beam/*.py apache_beam/coders apache_beam/internal apache_beam/ml apache_beam/options apache_beam/portability apache_beam/testing apache_beam/tools apache_beam/typehints apache_beam/utils\"' // All other tests are covered by different jobs. + '-Pposargs=\"apache_beam/*.py apache_beam/coders apache_beam/internal apache_beam/io apache_beam/metrics apache_beam/ml apache_beam/options apache_beam/portability apache_beam/testing apache_beam/tools apache_beam/typehints apache_beam/utils\"' // All other tests are covered by different jobs. ], timeoutMins: 180, triggerPathPatterns: [ From c3a6797f777137312387f671657c796de5ba3b6f Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 28 Dec 2022 11:25:36 -0500 Subject: [PATCH 23/23] Move arg lines down --- sdks/python/test-suites/tox/py38/build.gradle | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 91d06cddf2e9..9a28a4c96dcb 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -26,10 +26,6 @@ applyPythonNature() // Required to setup a Python 3 virtualenv and task names. pythonVersion = '3.8' -def posargs = project.findProperty("posargs") ?: "" -def runDataframesTests = (posargs == "" || posargs.contains("dataframes")) -logger.info('Running dataframes tests: ' + runDataframesTests) - toxTask "formatter", "py3-yapf-check" check.dependsOn formatter @@ -38,8 +34,12 @@ apply from: "../common.gradle" // TODO(https://github.com/apache/beam/issues/20051): Remove this once tox uses isolated builds. testPy38Cython.mustRunAfter testPython38, testPy38CloudCoverage +// TODO(BEAM-12000): Move tasks that aren't specific to 3.8 to Py 3.9. +def posargs = project.findProperty("posargs") ?: "" +def runDataframesTests = (posargs == "" || posargs.contains("dataframes")) +logger.info('Running dataframes tests: ' + runDataframesTests) + // Create a test task for each major version of pyarrow -// TODO(BEAM-12000): Move these to Py 3.9. toxTask "testPy38pyarrow-0", "py38-pyarrow-0", "${posargs}" test.dependsOn "testPy38pyarrow-0" preCommitPy38.dependsOn "testPy38pyarrow-0"