From 3080704106f342ecc2e6e73bef40f7c8021ef7ef Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Fri, 9 Sep 2022 14:29:11 -0500 Subject: [PATCH 1/6] Migrate Python Examples Jobs Postcommit from Jenkins to GA --- ...ob-postcommit-python-examples-dataflow.yml | 45 +++++++++++++++++++ .../job-postcommit-python-examples-direct.yml | 45 +++++++++++++++++++ .../job-postcommit-python-examples-flink.yml | 45 +++++++++++++++++++ .../job-postcommit-python-examples-spark.yml | 45 +++++++++++++++++++ .test-infra/jenkins/README.md | 3 -- ...PostCommit_Python_Examples_Dataflow.groovy | 43 ------------------ ...b_PostCommit_Python_Examples_Direct.groovy | 43 ------------------ ...ob_PostCommit_Python_Examples_Flink.groovy | 43 ------------------ ...ob_PostCommit_Python_Examples_Spark.groovy | 45 ------------------- CI.md | 22 +++++++++ 10 files changed, 202 insertions(+), 177 deletions(-) create mode 100644 .github/workflows/job-postcommit-python-examples-dataflow.yml create mode 100644 .github/workflows/job-postcommit-python-examples-direct.yml create mode 100644 .github/workflows/job-postcommit-python-examples-flink.yml create mode 100644 .github/workflows/job-postcommit-python-examples-spark.yml delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_Examples_Dataflow.groovy delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_Examples_Direct.groovy delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_Examples_Spark.groovy diff --git a/.github/workflows/job-postcommit-python-examples-dataflow.yml b/.github/workflows/job-postcommit-python-examples-dataflow.yml new file mode 100644 index 000000000000..ffa07e6e2ed3 --- /dev/null +++ b/.github/workflows/job-postcommit-python-examples-dataflow.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the Python examples tests with DataflowRunner. + +name: Python Examples Dataflow + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + +jobs: + python-examples-dataflow: + name: Python Dataflow Runner Examples + runs-on: self-hosted + timeout-minutes: 120 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:python:test-suites:dataflow:examplesPostCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:dataflow:examplesPostCommit diff --git a/.github/workflows/job-postcommit-python-examples-direct.yml b/.github/workflows/job-postcommit-python-examples-direct.yml new file mode 100644 index 000000000000..8585b98385eb --- /dev/null +++ b/.github/workflows/job-postcommit-python-examples-direct.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the Python examples tests with DirectRunner. + +name: Python Examples Direct + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + +jobs: + python-examples-direct: + name: Python Direct Runner Examples + runs-on: self-hosted + timeout-minutes: 120 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:python:test-suites:direct:examplesPostCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:direct:examplesPostCommit diff --git a/.github/workflows/job-postcommit-python-examples-flink.yml b/.github/workflows/job-postcommit-python-examples-flink.yml new file mode 100644 index 000000000000..f755401ecf4a --- /dev/null +++ b/.github/workflows/job-postcommit-python-examples-flink.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the Python examples tests with FlinkRunner. + +name: Python Examples Flink + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + +jobs: + python-examples-flink: + name: Python Flink Runner Examples + runs-on: self-hosted + timeout-minutes: 120 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:python:test-suites:portable:flinkExamplesPostCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:flinkExamplesPostCommit \ No newline at end of file diff --git a/.github/workflows/job-postcommit-python-examples-spark.yml b/.github/workflows/job-postcommit-python-examples-spark.yml new file mode 100644 index 000000000000..7bc14a39ff71 --- /dev/null +++ b/.github/workflows/job-postcommit-python-examples-spark.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the Python examples tests with SparkRunner. + +name: Python Examples Spark + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + +jobs: + python-examples-spark: + name: Python Spark Runner Examples + runs-on: self-hosted + timeout-minutes: 120 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:python:test-suites:portable:sparkExamplesPostCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:sparkExamplesPostCommit \ No newline at end of file diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 2c252a4d55aa..522c543dc06c 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -117,9 +117,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Py_ValCont | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_PR/) | `Run Python Dataflow ValidatesContainer` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont) | | beam_PostCommit_Python_VR_Flink | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Flink/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Flink_PR/) | `Run Python Flink ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Flink/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Flink) | | beam_PostCommit_Python_VR_Samza | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Samza/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Samza_PR/) | `Run Python Samza ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Samza/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Samza) | -| beam_PostCommit_Python_Examples_Direct | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct_PR/) | `Run Python Examples_Direct` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/) | -| beam_PostCommit_Python_Examples_Flink | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink_PR/) | `Run Python Examples_Flink` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink/) | -| beam_PostCommit_Python_Examples_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark_PR/) | `Run Python Examples_Spark` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/) | | beam_PostCommit_Python_Chicago_Taxi_Example_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow_PR/) | `Run Chicago Taxi on Dataflow` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow) | | beam_PostCommit_Python_Chicago_Taxi_Example_Flink | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Flink/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Flink_PR/) | `Run Chicago Taxi on Flink` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Flink/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Flink) | | beam_PostCommit_Python_MongoDBIO_IT | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_MongoDBIO_IT), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_MongoDBIO_IT_PR/) | `Run Python MongoDBIO_IT` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_MongoDBIO_IT/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_MongoDBIO_IT) | diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Dataflow.groovy deleted file mode 100644 index 402402504c92..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_Examples_Dataflow.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the Python examples tests with DataflowRunner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Dataflow', - 'Run Python Examples_Dataflow', 'Python Dataflow Runner Examples', this) { - - description('Runs the Python Examples with DataflowRunner') - - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/pytest*.xml') - } - - // Execute shell command to run examples. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":sdks:python:test-suites:dataflow:examplesPostCommit") - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Direct.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Direct.groovy deleted file mode 100644 index e420b046271d..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_Examples_Direct.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the Python examples tests with DirectRunner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Direct', - 'Run Python Examples_Direct', 'Python Direct Runner Examples', this) { - - description('Runs the Python Examples with DirectRunner') - - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/pytest*.xml') - } - - // Execute shell command to run examples. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":sdks:python:test-suites:direct:examplesPostCommit") - commonJobProperties.setGradleSwitches(delegate) - } - } - } \ No newline at end of file diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy deleted file mode 100644 index 779395bf7093..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the Python examples tests with FlinkRunner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Flink', - 'Run Python Examples_Flink', 'Python Flink Runner Examples', this) { - - description('Runs the Python Examples with Flink Runner') - - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/pytest*.xml') - } - - // Execute shell command to run examples. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":sdks:python:test-suites:portable:flinkExamplesPostCommit") - commonJobProperties.setGradleSwitches(delegate) - } - } - } \ No newline at end of file diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Spark.groovy deleted file mode 100644 index 6277106c8cbc..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_Examples_Spark.groovy +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the Python examples tests with SparkRunner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Spark', - 'Run Python Examples_Spark', 'Python Spark Runner Examples', this) { - - description('Runs the Python Examples with Spark Runner') - - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/pytest*.xml') - } - - // Execute shell command to run examples. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":sdks:python:test-suites:portable:sparkExamplesPostCommit") - commonJobProperties.setGradleSwitches(delegate) - } - } - } - - diff --git a/CI.md b/CI.md index ed75a4dd4433..d006bf944422 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,28 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PreCommit Workflows + +#### Python Examples Dataflow - [job-postcommit-python-examples-dataflow.yml](.github/workflows/job-postcommit-python-examples-dataflow.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|---------------------------------|--------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Dataflow Runner Examples | Runs Python Dataflow Runner Examples | - | Yes | Yes | - | + +#### Python Examples Direct - [job-postcommit-python-examples-direct.yml](.github/workflows/job-postcommit-python-examples-direct.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-------------------------------|------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Direct Runner Examples | Runs Python Direct Runner Examples | - | Yes | Yes | - | + +#### Python Examples Flink - [job-postcommit-python-examples-flink.yml](.github/workflows/job-postcommit-python-examples-flink.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------------------|-----------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Flink Runner Examples | Runs Python Flink Runner Examples | - | Yes | Yes | - | + +#### Python Examples Spark - [job-postcommit-python-examples-spark.yml](.github/workflows/job-postcommit-python-examples-spark.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------------------|-----------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Spark Runner Examples | Runs Python Spark Runner Examples | - | Yes | Yes | - | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From 9e035c0f7409ad72c0077a1b9fb67afe76011374 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez <74670721+benWize@users.noreply.github.com> Date: Mon, 12 Sep 2022 11:34:41 -0500 Subject: [PATCH 2/6] Update CI.md --- CI.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI.md b/CI.md index d006bf944422..6669ac8a4945 100644 --- a/CI.md +++ b/CI.md @@ -125,7 +125,7 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | -### PreCommit Workflows +### PostCommit Workflows #### Python Examples Dataflow - [job-postcommit-python-examples-dataflow.yml](.github/workflows/job-postcommit-python-examples-dataflow.yml) | Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | From acc028ceccc16db23598538330bf64866f559157 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 15 Sep 2022 14:30:07 -0500 Subject: [PATCH 3/6] Use optional go installation in setup --- .../workflows/job-postcommit-python-examples-dataflow.yml | 8 ++++++-- .../workflows/job-postcommit-python-examples-direct.yml | 8 ++++++-- .../workflows/job-postcommit-python-examples-flink.yml | 8 ++++++-- .../workflows/job-postcommit-python-examples-spark.yml | 8 ++++++-- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/job-postcommit-python-examples-dataflow.yml b/.github/workflows/job-postcommit-python-examples-dataflow.yml index ffa07e6e2ed3..798dffb98941 100644 --- a/.github/workflows/job-postcommit-python-examples-dataflow.yml +++ b/.github/workflows/job-postcommit-python-examples-dataflow.yml @@ -20,11 +20,13 @@ name: Python Examples Dataflow on: + workflow_dispatch: schedule: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] +permissions: read-all jobs: python-examples-dataflow: @@ -33,12 +35,14 @@ jobs: timeout-minutes: 120 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false - name: Run :sdks:python:test-suites:dataflow:examplesPostCommit uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-postcommit-python-examples-direct.yml b/.github/workflows/job-postcommit-python-examples-direct.yml index 8585b98385eb..0d237e477d5f 100644 --- a/.github/workflows/job-postcommit-python-examples-direct.yml +++ b/.github/workflows/job-postcommit-python-examples-direct.yml @@ -20,11 +20,13 @@ name: Python Examples Direct on: + workflow_dispatch: schedule: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] +permissions: read-all jobs: python-examples-direct: @@ -33,12 +35,14 @@ jobs: timeout-minutes: 120 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false - name: Run :sdks:python:test-suites:direct:examplesPostCommit uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-postcommit-python-examples-flink.yml b/.github/workflows/job-postcommit-python-examples-flink.yml index f755401ecf4a..a0dfec59cbe8 100644 --- a/.github/workflows/job-postcommit-python-examples-flink.yml +++ b/.github/workflows/job-postcommit-python-examples-flink.yml @@ -20,11 +20,13 @@ name: Python Examples Flink on: + workflow_dispatch: schedule: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] +permissions: read-all jobs: python-examples-flink: @@ -33,12 +35,14 @@ jobs: timeout-minutes: 120 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false - name: Run :sdks:python:test-suites:portable:flinkExamplesPostCommit uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-postcommit-python-examples-spark.yml b/.github/workflows/job-postcommit-python-examples-spark.yml index 7bc14a39ff71..dab460456670 100644 --- a/.github/workflows/job-postcommit-python-examples-spark.yml +++ b/.github/workflows/job-postcommit-python-examples-spark.yml @@ -20,11 +20,13 @@ name: Python Examples Spark on: + workflow_dispatch: schedule: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] +permissions: read-all jobs: python-examples-spark: @@ -33,12 +35,14 @@ jobs: timeout-minutes: 120 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false - name: Run :sdks:python:test-suites:portable:sparkExamplesPostCommit uses: ./.github/actions/gradle-command-self-hosted-action with: From 910c3258fce2677bbd2da5464c443a9c2ac545b8 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Fri, 23 Sep 2022 14:26:06 -0500 Subject: [PATCH 4/6] Change from self-hosted to [self-hosted, ubuntu-20.04] --- .github/workflows/job-postcommit-python-examples-dataflow.yml | 2 +- .github/workflows/job-postcommit-python-examples-direct.yml | 2 +- .github/workflows/job-postcommit-python-examples-flink.yml | 2 +- .github/workflows/job-postcommit-python-examples-spark.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/job-postcommit-python-examples-dataflow.yml b/.github/workflows/job-postcommit-python-examples-dataflow.yml index 798dffb98941..7b9e17e6045f 100644 --- a/.github/workflows/job-postcommit-python-examples-dataflow.yml +++ b/.github/workflows/job-postcommit-python-examples-dataflow.yml @@ -31,7 +31,7 @@ permissions: read-all jobs: python-examples-dataflow: name: Python Dataflow Runner Examples - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 120 steps: - name: Checkout code diff --git a/.github/workflows/job-postcommit-python-examples-direct.yml b/.github/workflows/job-postcommit-python-examples-direct.yml index 0d237e477d5f..402a27ee2f76 100644 --- a/.github/workflows/job-postcommit-python-examples-direct.yml +++ b/.github/workflows/job-postcommit-python-examples-direct.yml @@ -31,7 +31,7 @@ permissions: read-all jobs: python-examples-direct: name: Python Direct Runner Examples - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 120 steps: - name: Checkout code diff --git a/.github/workflows/job-postcommit-python-examples-flink.yml b/.github/workflows/job-postcommit-python-examples-flink.yml index a0dfec59cbe8..846a31f881ca 100644 --- a/.github/workflows/job-postcommit-python-examples-flink.yml +++ b/.github/workflows/job-postcommit-python-examples-flink.yml @@ -31,7 +31,7 @@ permissions: read-all jobs: python-examples-flink: name: Python Flink Runner Examples - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 120 steps: - name: Checkout code diff --git a/.github/workflows/job-postcommit-python-examples-spark.yml b/.github/workflows/job-postcommit-python-examples-spark.yml index dab460456670..9d96261b96e2 100644 --- a/.github/workflows/job-postcommit-python-examples-spark.yml +++ b/.github/workflows/job-postcommit-python-examples-spark.yml @@ -31,7 +31,7 @@ permissions: read-all jobs: python-examples-spark: name: Python Spark Runner Examples - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 120 steps: - name: Checkout code From 55daf44c42413c659ae94b9d2d5c99395337478c Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 29 Sep 2022 17:11:39 -0500 Subject: [PATCH 5/6] Remove submodules:recursive --- .github/workflows/job-postcommit-python-examples-dataflow.yml | 1 - .github/workflows/job-postcommit-python-examples-direct.yml | 1 - .github/workflows/job-postcommit-python-examples-flink.yml | 1 - .github/workflows/job-postcommit-python-examples-spark.yml | 1 - 4 files changed, 4 deletions(-) diff --git a/.github/workflows/job-postcommit-python-examples-dataflow.yml b/.github/workflows/job-postcommit-python-examples-dataflow.yml index 7b9e17e6045f..e700b5ec6bd1 100644 --- a/.github/workflows/job-postcommit-python-examples-dataflow.yml +++ b/.github/workflows/job-postcommit-python-examples-dataflow.yml @@ -38,7 +38,6 @@ jobs: uses: actions/checkout@v3 with: persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-postcommit-python-examples-direct.yml b/.github/workflows/job-postcommit-python-examples-direct.yml index 402a27ee2f76..a1dcc1ccfe50 100644 --- a/.github/workflows/job-postcommit-python-examples-direct.yml +++ b/.github/workflows/job-postcommit-python-examples-direct.yml @@ -38,7 +38,6 @@ jobs: uses: actions/checkout@v3 with: persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-postcommit-python-examples-flink.yml b/.github/workflows/job-postcommit-python-examples-flink.yml index 846a31f881ca..b77b18bf2806 100644 --- a/.github/workflows/job-postcommit-python-examples-flink.yml +++ b/.github/workflows/job-postcommit-python-examples-flink.yml @@ -38,7 +38,6 @@ jobs: uses: actions/checkout@v3 with: persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-postcommit-python-examples-spark.yml b/.github/workflows/job-postcommit-python-examples-spark.yml index 9d96261b96e2..f9c8202cc6b3 100644 --- a/.github/workflows/job-postcommit-python-examples-spark.yml +++ b/.github/workflows/job-postcommit-python-examples-spark.yml @@ -38,7 +38,6 @@ jobs: uses: actions/checkout@v3 with: persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: From 6e3b7b7a4d68f674ea52f20a94bdf9628341a91c Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Fri, 9 Dec 2022 16:48:04 -0600 Subject: [PATCH 6/6] Setup service account with action --- .github/workflows/job-postcommit-python-examples-dataflow.yml | 4 ++++ .github/workflows/job-postcommit-python-examples-direct.yml | 4 ++++ .github/workflows/job-postcommit-python-examples-flink.yml | 4 ++++ .github/workflows/job-postcommit-python-examples-spark.yml | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/.github/workflows/job-postcommit-python-examples-dataflow.yml b/.github/workflows/job-postcommit-python-examples-dataflow.yml index e700b5ec6bd1..1d8cdedac789 100644 --- a/.github/workflows/job-postcommit-python-examples-dataflow.yml +++ b/.github/workflows/job-postcommit-python-examples-dataflow.yml @@ -42,6 +42,10 @@ jobs: uses: ./.github/actions/setup-self-hosted-action with: requires-go-18: false + - name: Set up service account + uses: ./.github/actions/activate-service-account + with: + GCP_SERVICE_ACCOUNT: ${{secrets.GCP_SELF_HOSTED_SA_KEY}} - name: Run :sdks:python:test-suites:dataflow:examplesPostCommit uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-postcommit-python-examples-direct.yml b/.github/workflows/job-postcommit-python-examples-direct.yml index a1dcc1ccfe50..0952d55f81f8 100644 --- a/.github/workflows/job-postcommit-python-examples-direct.yml +++ b/.github/workflows/job-postcommit-python-examples-direct.yml @@ -42,6 +42,10 @@ jobs: uses: ./.github/actions/setup-self-hosted-action with: requires-go-18: false + - name: Set up service account + uses: ./.github/actions/activate-service-account + with: + GCP_SERVICE_ACCOUNT: ${{secrets.GCP_SELF_HOSTED_SA_KEY}} - name: Run :sdks:python:test-suites:direct:examplesPostCommit uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-postcommit-python-examples-flink.yml b/.github/workflows/job-postcommit-python-examples-flink.yml index b77b18bf2806..c1c667ee3926 100644 --- a/.github/workflows/job-postcommit-python-examples-flink.yml +++ b/.github/workflows/job-postcommit-python-examples-flink.yml @@ -42,6 +42,10 @@ jobs: uses: ./.github/actions/setup-self-hosted-action with: requires-go-18: false + - name: Set up service account + uses: ./.github/actions/activate-service-account + with: + GCP_SERVICE_ACCOUNT: ${{secrets.GCP_SELF_HOSTED_SA_KEY}} - name: Run :sdks:python:test-suites:portable:flinkExamplesPostCommit uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-postcommit-python-examples-spark.yml b/.github/workflows/job-postcommit-python-examples-spark.yml index f9c8202cc6b3..6ee4d3b04880 100644 --- a/.github/workflows/job-postcommit-python-examples-spark.yml +++ b/.github/workflows/job-postcommit-python-examples-spark.yml @@ -42,6 +42,10 @@ jobs: uses: ./.github/actions/setup-self-hosted-action with: requires-go-18: false + - name: Set up service account + uses: ./.github/actions/activate-service-account + with: + GCP_SERVICE_ACCOUNT: ${{secrets.GCP_SELF_HOSTED_SA_KEY}} - name: Run :sdks:python:test-suites:portable:sparkExamplesPostCommit uses: ./.github/actions/gradle-command-self-hosted-action with: