From 010d1d3322be851c734827ea5756c974cd6fd862 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Sun, 22 Jan 2023 20:43:40 -0500 Subject: [PATCH 1/2] Fix Python PreCommit dependency compatibility test not run or redundant setup * Move pyarrow and pandas matrix unit test to preCommitPyCoverage * Fix Dataframe compatibility test not run * Update readme --- .test-infra/jenkins/README.md | 1 + sdks/python/test-suites/tox/py38/build.gradle | 42 ++++++++++--------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 0b7b20f5433f..a99d4b0c9bd7 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -47,6 +47,7 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PreCommit_Python_Examples | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Phrase/) | `Run Python_Examples PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Examples_Cron) | | beam_PreCommit_Python_Runners | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Phrase/) | `Run Python_Runners PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Runners_Cron) | | beam_PreCommit_Python_Transforms | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Phrase/) | `Run Python_Transforms PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Transforms_Cron) | +| beam_PreCommit_Python_Coverage | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Coverage_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Coverage_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Coverage_Phrase/) | `Run Python_Coverage PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Coverage_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Coverage_Cron) | | beam_PreCommit_PythonDocker | [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Phrase/) | `Run PythonDocker PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Cron/) | | beam_PreCommit_PythonDocs| [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Phrase/) | `Run PythonDocs PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/) | | beam_PreCommit_Python_PVR_Flink | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Phrase/) | `Run Python_PVR_Flink PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron) | diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 072beb172cfd..2590e5b24973 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -36,52 +36,58 @@ testPy38Cython.mustRunAfter testPython38, testPy38CloudCoverage // TODO(BEAM-12000): Move tasks that aren't specific to 3.8 to Py 3.9. def posargs = project.findProperty("posargs") ?: "" -def runDataframesTests = (posargs == "" || posargs.contains("dataframes")) -logger.info('Running dataframes tests: ' + runDataframesTests) +def runDataframeTests = (posargs == "" || posargs.contains("dataframe")) +logger.info('Running dataframes tests: ' + runDataframeTests) + +// PyCoverage Precommit runs test suites that evaluate test coverage and compatibility of +// particular dependencies. It is exercised on a single Python version. +project.tasks.register("preCommitPyCoverage") { + dependsOn = ["testPy38CloudCoverage"] +} // Create a test task for each major version of pyarrow toxTask "testPy38pyarrow-0", "py38-pyarrow-0", "${posargs}" test.dependsOn "testPy38pyarrow-0" -preCommitPy38.dependsOn "testPy38pyarrow-0" +preCommitPyCoverage.dependsOn "testPy38pyarrow-0" toxTask "testPy38pyarrow-1", "py38-pyarrow-1", "${posargs}" test.dependsOn "testPy38pyarrow-1" -preCommitPy38.dependsOn "testPy38pyarrow-1" +preCommitPyCoverage.dependsOn "testPy38pyarrow-1" toxTask "testPy38pyarrow-2", "py38-pyarrow-2", "${posargs}" test.dependsOn "testPy38pyarrow-2" -preCommitPy38.dependsOn "testPy38pyarrow-2" +preCommitPyCoverage.dependsOn "testPy38pyarrow-2" toxTask "testPy38pyarrow-3", "py38-pyarrow-3", "${posargs}" test.dependsOn "testPy38pyarrow-3" -preCommitPy38.dependsOn "testPy38pyarrow-3" +preCommitPyCoverage.dependsOn "testPy38pyarrow-3" toxTask "testPy38pyarrow-4", "py38-pyarrow-4", "${posargs}" test.dependsOn "testPy38pyarrow-4" -preCommitPy38.dependsOn "testPy38pyarrow-4" +preCommitPyCoverage.dependsOn "testPy38pyarrow-4" toxTask "testPy38pyarrow-5", "py38-pyarrow-5", "${posargs}" test.dependsOn "testPy38pyarrow-5" -preCommitPy38.dependsOn "testPy38pyarrow-5" +preCommitPyCoverage.dependsOn "testPy38pyarrow-5" toxTask "testPy38pyarrow-6", "py38-pyarrow-6", "${posargs}" test.dependsOn "testPy38pyarrow-6" -preCommitPy38.dependsOn "testPy38pyarrow-6" +preCommitPyCoverage.dependsOn "testPy38pyarrow-6" toxTask "testPy38pyarrow-7", "py38-pyarrow-7", "${posargs}" test.dependsOn "testPy38pyarrow-7" -preCommitPy38.dependsOn "testPy38pyarrow-7" +preCommitPyCoverage.dependsOn "testPy38pyarrow-7" toxTask "testPy38pyarrow-8", "py38-pyarrow-8", "${posargs}" test.dependsOn "testPy38pyarrow-8" -preCommitPy38.dependsOn "testPy38pyarrow-8" +preCommitPyCoverage.dependsOn "testPy38pyarrow-8" toxTask "testPy38pyarrow-9", "py38-pyarrow-9", "${posargs}" test.dependsOn "testPy38pyarrow-9" -preCommitPy38.dependsOn "testPy38pyarrow-9" +preCommitPyCoverage.dependsOn "testPy38pyarrow-9" -// Dataframes tests won't get auto-filtered by path since they explicitly only run on the dataframes directory in tox.ini -if (runDataframesTests) { +// Dataframe tests won't get auto-filtered by path since they explicitly only run on the dataframes directory in tox.ini +if (runDataframeTests) { // Create a test task for each minor version of pandas toxTask "testPy38pandas-11", "py38-pandas-11", "${posargs}" test.dependsOn "testPy38pandas-11" @@ -107,11 +113,11 @@ if (runDataframesTests) { // Create a test task for each minor version of pytorch toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}" test.dependsOn "testPy38pytorch-19" -preCommitPy38.dependsOn "testPy38pytorch-19" +preCommitPyCoverage.dependsOn "testPy38pytorch-19" toxTask "testPy38pytorch-110", "py38-pytorch-110", "${posargs}" test.dependsOn "testPy38pytorch-110" -preCommitPy38.dependsOn "testPy38pytorch-110" +preCommitPyCoverage.dependsOn "testPy38pytorch-110" toxTask "whitespacelint", "whitespacelint", "${posargs}" @@ -126,10 +132,6 @@ task archiveFilesToLint(type: Zip) { } } -project.tasks.register("preCommitPyCoverage") { - dependsOn = ["testPy38CloudCoverage"] -} - task unpackFilesToLint(type: Copy) { from zipTree("$buildDir/dist/files-to-whitespacelint.zip") into "$buildDir/files-to-whitespacelint" From 9fc7ef06de8369af8f0af7c5493f6d59f513869f Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Mon, 23 Jan 2023 09:58:40 -0500 Subject: [PATCH 2/2] Also move dataframe comp tests to preCommitPyCoverage --- sdks/python/test-suites/tox/py38/build.gradle | 37 ++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 2590e5b24973..e90b306f698f 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -36,8 +36,6 @@ testPy38Cython.mustRunAfter testPython38, testPy38CloudCoverage // TODO(BEAM-12000): Move tasks that aren't specific to 3.8 to Py 3.9. def posargs = project.findProperty("posargs") ?: "" -def runDataframeTests = (posargs == "" || posargs.contains("dataframe")) -logger.info('Running dataframes tests: ' + runDataframeTests) // PyCoverage Precommit runs test suites that evaluate test coverage and compatibility of // particular dependencies. It is exercised on a single Python version. @@ -86,29 +84,26 @@ toxTask "testPy38pyarrow-9", "py38-pyarrow-9", "${posargs}" test.dependsOn "testPy38pyarrow-9" preCommitPyCoverage.dependsOn "testPy38pyarrow-9" -// Dataframe tests won't get auto-filtered by path since they explicitly only run on the dataframes directory in tox.ini -if (runDataframeTests) { - // Create a test task for each minor version of pandas - toxTask "testPy38pandas-11", "py38-pandas-11", "${posargs}" - test.dependsOn "testPy38pandas-11" - preCommitPy38.dependsOn "testPy38pandas-11" +// Create a test task for each minor version of pandas +toxTask "testPy38pandas-11", "py38-pandas-11", "${posargs}" +test.dependsOn "testPy38pandas-11" +preCommitPyCoverage.dependsOn "testPy38pandas-11" - toxTask "testPy38pandas-12", "py38-pandas-12", "${posargs}" - test.dependsOn "testPy38pandas-12" - preCommitPy38.dependsOn "testPy38pandas-12" +toxTask "testPy38pandas-12", "py38-pandas-12", "${posargs}" +test.dependsOn "testPy38pandas-12" +preCommitPyCoverage.dependsOn "testPy38pandas-12" - toxTask "testPy38pandas-13", "py38-pandas-13", "${posargs}" - test.dependsOn "testPy38pandas-13" - preCommitPy38.dependsOn "testPy38pandas-13" +toxTask "testPy38pandas-13", "py38-pandas-13", "${posargs}" +test.dependsOn "testPy38pandas-13" +preCommitPyCoverage.dependsOn "testPy38pandas-13" - toxTask "testPy38pandas-14", "py38-pandas-14", "${posargs}" - test.dependsOn "testPy38pandas-14" - preCommitPy38.dependsOn "testPy38pandas-14" +toxTask "testPy38pandas-14", "py38-pandas-14", "${posargs}" +test.dependsOn "testPy38pandas-14" +preCommitPyCoverage.dependsOn "testPy38pandas-14" - toxTask "testPy38pandas-15", "py38-pandas-15", "${posargs}" - test.dependsOn "testPy38pandas-15" - preCommitPy38.dependsOn "testPy38pandas-15" -} +toxTask "testPy38pandas-15", "py38-pandas-15", "${posargs}" +test.dependsOn "testPy38pandas-15" +preCommitPyCoverage.dependsOn "testPy38pandas-15" // Create a test task for each minor version of pytorch toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}"