From f78ffdfdaf92ff5ad1d6701f224b60786421380e Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Thu, 11 Aug 2022 11:37:54 -0500 Subject: [PATCH 01/14] add portablejar flink postcommit job and add go setup to action --- .../job-postcommit-portablejar-flink.yml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/job-postcommit-portablejar-flink.yml diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml new file mode 100644 index 000000000000..173cc3b03c78 --- /dev/null +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: PortableJar Flink PostCommit +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: '^sdks/python/test-suites/portable/*' + +jobs: + java_vr_dataflow_v2: + name: Run PortableJar_Flink PostCommit + runs-on: self-hosted + strategy: + fail-fast: false + timeout-minutes: 120 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Gradle Test Pipeline Jar Flink Runner + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunnerw \ No newline at end of file From b32091d33f9e9977f2276b48de38c0d5daea378e Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Thu, 11 Aug 2022 11:41:35 -0500 Subject: [PATCH 02/14] comment path for postcommit --- .github/workflows/job-postcommit-portablejar-flink.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml index 173cc3b03c78..aacc7335fda1 100644 --- a/.github/workflows/job-postcommit-portablejar-flink.yml +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -25,7 +25,7 @@ on: pull_request: branches: ['master', 'release-*'] tags: 'v*' - paths: '^sdks/python/test-suites/portable/*' + # paths: '^sdks/python/test-suites/portable/*' jobs: java_vr_dataflow_v2: From 4d7df2ec090a3ba072571e2ab9ef4ff225f6c529 Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Thu, 11 Aug 2022 11:43:27 -0500 Subject: [PATCH 03/14] modify wrong gradle command --- .github/workflows/job-postcommit-portablejar-flink.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml index aacc7335fda1..0016ed87968e 100644 --- a/.github/workflows/job-postcommit-portablejar-flink.yml +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -43,4 +43,4 @@ jobs: - name: Gradle Test Pipeline Jar Flink Runner uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunnerw \ No newline at end of file + gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner \ No newline at end of file From 7f0236bb565d8274046ad24f6b1806482f133fe4 Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Thu, 11 Aug 2022 12:24:03 -0500 Subject: [PATCH 04/14] modify docker confg --- .../fnexecution/environment/DockerEnvironmentFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index ee816a944e5a..70004e4d386b 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -182,7 +182,7 @@ private List gcsCredentialArgs() { // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually. if (Files.exists(Paths.get(localGcloudConfig))) { return ImmutableList.of( - "--mount", + "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); } else { return ImmutableList.of(); From cbf0b8d514aa547b5cdc3ac8f0474345aaaf4d4f Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Thu, 11 Aug 2022 14:08:41 -0500 Subject: [PATCH 05/14] add default gradle switches --- .github/workflows/job-postcommit-portablejar-flink.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml index 0016ed87968e..dccc22c0e80c 100644 --- a/.github/workflows/job-postcommit-portablejar-flink.yml +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -43,4 +43,5 @@ jobs: - name: Gradle Test Pipeline Jar Flink Runner uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner \ No newline at end of file + gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner + gradle-switches: --continue \ No newline at end of file From 015ea5270fd69e1f01c21988507499872dfe0d6d Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Thu, 11 Aug 2022 14:24:11 -0500 Subject: [PATCH 06/14] add default switch for docker (test) --- .github/workflows/job-postcommit-portablejar-flink.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml index dccc22c0e80c..0fcd221060ce 100644 --- a/.github/workflows/job-postcommit-portablejar-flink.yml +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -44,4 +44,4 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner - gradle-switches: --continue \ No newline at end of file + gradle-switches: --continue -Pdocker-pull-licenses \ No newline at end of file From a0aecc40b7181958710446f560860c8b6ebf19f7 Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Thu, 11 Aug 2022 15:02:10 -0500 Subject: [PATCH 07/14] add portablejar spark job --- .../job-postcommit-portablejar-spark.yml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/job-postcommit-portablejar-spark.yml diff --git a/.github/workflows/job-postcommit-portablejar-spark.yml b/.github/workflows/job-postcommit-portablejar-spark.yml new file mode 100644 index 000000000000..4e70d8b08f3b --- /dev/null +++ b/.github/workflows/job-postcommit-portablejar-spark.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: PortableJar Spark PostCommit +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + # paths: '^sdks/python/test-suites/portable/*' + +jobs: + java_vr_dataflow_v2: + name: Run PortableJar Spark PostCommit + runs-on: self-hosted + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Gradle Test Pipeline Jar Spark Runner + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarSparkRunner + gradle-switchs: --continue -Pdocker-pull-licenses \ No newline at end of file From e95e53cacc8770c6b9eec23b75ed4f4c60811ab5 Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Tue, 23 Aug 2022 13:03:56 -0500 Subject: [PATCH 08/14] update workflow with base action --- .../workflows/job-postcommit-portablejar-flink.yml | 12 +++++------- .../workflows/job-postcommit-portablejar-spark.yml | 10 +++++----- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml index 0fcd221060ce..e0a75fefd1b9 100644 --- a/.github/workflows/job-postcommit-portablejar-flink.yml +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -25,14 +25,11 @@ on: pull_request: branches: ['master', 'release-*'] tags: 'v*' - # paths: '^sdks/python/test-suites/portable/*' jobs: - java_vr_dataflow_v2: + portablejar_flink: name: Run PortableJar_Flink PostCommit runs-on: self-hosted - strategy: - fail-fast: false timeout-minutes: 120 steps: - name: Checkout code @@ -40,8 +37,9 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Gradle Test Pipeline Jar Flink Runner + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run Pipeline Jar Flink Runner uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner - gradle-switches: --continue -Pdocker-pull-licenses \ No newline at end of file + gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner \ No newline at end of file diff --git a/.github/workflows/job-postcommit-portablejar-spark.yml b/.github/workflows/job-postcommit-portablejar-spark.yml index 4e70d8b08f3b..1cc330e40cba 100644 --- a/.github/workflows/job-postcommit-portablejar-spark.yml +++ b/.github/workflows/job-postcommit-portablejar-spark.yml @@ -25,10 +25,9 @@ on: pull_request: branches: ['master', 'release-*'] tags: 'v*' - # paths: '^sdks/python/test-suites/portable/*' jobs: - java_vr_dataflow_v2: + portablejar_spark: name: Run PortableJar Spark PostCommit runs-on: self-hosted steps: @@ -37,8 +36,9 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Gradle Test Pipeline Jar Spark Runner + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run Pipeline Jar Spark Runner uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarSparkRunner - gradle-switchs: --continue -Pdocker-pull-licenses \ No newline at end of file + gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarSparkRunner \ No newline at end of file From 1ff11e8be91b2a90c7367c7e619cf7092e87aea4 Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Mon, 19 Sep 2022 12:11:06 -0500 Subject: [PATCH 09/14] update yml job postcommit --- .github/workflows/job-postcommit-portablejar-flink.yml | 6 +++--- .github/workflows/job-postcommit-portablejar-spark.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml index e0a75fefd1b9..7ec0349652b6 100644 --- a/.github/workflows/job-postcommit-portablejar-flink.yml +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -22,9 +22,9 @@ on: workflow_dispatch: schedule: - cron: '0 */6 * * *' - pull_request: + push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] jobs: portablejar_flink: @@ -33,7 +33,7 @@ jobs: timeout-minutes: 120 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive diff --git a/.github/workflows/job-postcommit-portablejar-spark.yml b/.github/workflows/job-postcommit-portablejar-spark.yml index 1cc330e40cba..b9f43b5f477d 100644 --- a/.github/workflows/job-postcommit-portablejar-spark.yml +++ b/.github/workflows/job-postcommit-portablejar-spark.yml @@ -22,9 +22,9 @@ on: workflow_dispatch: schedule: - cron: '0 */6 * * *' - pull_request: + push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] jobs: portablejar_spark: @@ -32,7 +32,7 @@ jobs: runs-on: self-hosted steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive From 25f8cdb7b9b4cae7035b28680d5dad74a060b809 Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Wed, 28 Sep 2022 09:06:45 -0500 Subject: [PATCH 10/14] resolve CI conflicts and fix --- .test-infra/jenkins/README.md | 2 -- CI.md | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 08b9665a92ba..ac017d7f7c4d 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -110,8 +110,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Java_VR_Dataflow_V2 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2_PR/) | `Run Java Dataflow V2 ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2/) | | beam_PostCommit_Java_VR_Dataflow_V2_Streaming | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2_Streaming/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2_Streaming_PR/) | `Run Java Dataflow V2 ValidatesRunner Streaming` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2_Streaming/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2_Streaming/) | | beam_PostCommit_Javadoc | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Javadoc/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Javadoc_PR/) | `Run Javadoc PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Javadoc/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Javadoc) | -| beam_PostCommit_PortableJar_Flink | [cron](https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Flink/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Flink_PR/) | `Run PortableJar_Flink PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Flink/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Flink) | -| beam_PostCommit_PortableJar_Spark | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark_PR/) | `Run PortableJar_Spark PostCommit` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark/) | | beam_PostCommit_Py_VR_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_PR/) | `Run Python Dataflow ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow) | | beam_PostCommit_Py_VR_Dataflow_V2 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2_PR/) | `Run Python Dataflow V2 ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2) | | beam_PostCommit_Py_ValCont | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_PR/) | `Run Python Dataflow ValidatesContainer` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont) | diff --git a/CI.md b/CI.md index ee911584f7f0..ec6b7073f5e3 100644 --- a/CI.md +++ b/CI.md @@ -140,7 +140,8 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog ### PostCommit Workflows | Workflow | Description | Requires GCP Credentials | |------------------------------------------------------------------------------------|-------------------------|--------------------------| -| [job-postcommit-placeholder.yml](.github/workflows/job-postcommit-placeholder.yml) | Description placeholder | Yes/No | +| [job-postcommit-java-portablejar-spark.yml](.github/workflows/job-postcommit-java-portablejar-spark.yml) | Tests creation and execution of portable pipeline Jars on the Spark runner | Yes/No | +| [job-postcommit-java-portablejar-flink.yml](.github/workflows/job-postcommit-java-portablejar-flink.yml) | Tests creation and execution of portable pipeline Jars on the Flink runner | Yes/No | ### GitHub Action Tips From 0a1cdc6a27e702c1dbde95b7a80ea04887e07891 Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Mon, 19 Sep 2022 12:47:30 -0500 Subject: [PATCH 11/14] remove jenkins jobs --- .../job_PostCommit_PortableJar_Flink.groovy | 38 ------------------- .../job_PostCommit_PortableJar_Spark.groovy | 38 ------------------- 2 files changed, 76 deletions(-) delete mode 100644 .test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy delete mode 100644 .test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy deleted file mode 100644 index 1332b61ccb04..000000000000 --- a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// Tests creation and execution of portable pipeline Jars on the Flink runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Flink', - 'Run PortableJar_Flink PostCommit', 'Flink Portable Jar Tests', this) { - description('Tests creation and execution of portable pipeline Jars on the Flink runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120) - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy deleted file mode 100644 index 93e58af8979a..000000000000 --- a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// Tests creation and execution of portable pipeline Jars on the Spark runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Spark', - 'Run PortableJar_Spark PostCommit', 'Spark Portable Jar Tests', this) { - description('Tests creation and execution of portable pipeline Jars on the Spark runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:portable:py37:testPipelineJarSparkRunner') - commonJobProperties.setGradleSwitches(delegate) - } - } - } From 261c706a573d9fcc5bf1be3261a5338af04c8aaf Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Sat, 24 Sep 2022 11:15:13 -0500 Subject: [PATCH 12/14] update runners jobs --- .github/workflows/job-postcommit-portablejar-flink.yml | 2 +- .github/workflows/job-postcommit-portablejar-spark.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml index 7ec0349652b6..15c7fd9a4b0f 100644 --- a/.github/workflows/job-postcommit-portablejar-flink.yml +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -29,7 +29,7 @@ on: jobs: portablejar_flink: name: Run PortableJar_Flink PostCommit - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 120 steps: - name: Checkout code diff --git a/.github/workflows/job-postcommit-portablejar-spark.yml b/.github/workflows/job-postcommit-portablejar-spark.yml index b9f43b5f477d..fa1a128c99c7 100644 --- a/.github/workflows/job-postcommit-portablejar-spark.yml +++ b/.github/workflows/job-postcommit-portablejar-spark.yml @@ -29,7 +29,7 @@ on: jobs: portablejar_spark: name: Run PortableJar Spark PostCommit - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] steps: - name: Checkout code uses: actions/checkout@v3 From 0403242b08c2a68eea4fa8b62901accc964f73c3 Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Wed, 28 Sep 2022 09:11:22 -0500 Subject: [PATCH 13/14] add property to job yml --- .github/workflows/job-postcommit-portablejar-flink.yml | 1 + .github/workflows/job-postcommit-portablejar-spark.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml index 15c7fd9a4b0f..32e4cbbf2422 100644 --- a/.github/workflows/job-postcommit-portablejar-flink.yml +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -25,6 +25,7 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] +permissions: read-all jobs: portablejar_flink: diff --git a/.github/workflows/job-postcommit-portablejar-spark.yml b/.github/workflows/job-postcommit-portablejar-spark.yml index fa1a128c99c7..b45fb22e07c3 100644 --- a/.github/workflows/job-postcommit-portablejar-spark.yml +++ b/.github/workflows/job-postcommit-portablejar-spark.yml @@ -25,6 +25,7 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] +permissions: read-all jobs: portablejar_spark: From c5188fa1082c798626725d5ff2571fc8928bb5b0 Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Fri, 30 Sep 2022 12:11:10 -0500 Subject: [PATCH 14/14] remove yml properties --- .github/workflows/job-postcommit-portablejar-flink.yml | 1 - .github/workflows/job-postcommit-portablejar-spark.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml index 32e4cbbf2422..7025982882da 100644 --- a/.github/workflows/job-postcommit-portablejar-flink.yml +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -37,7 +37,6 @@ jobs: uses: actions/checkout@v3 with: persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action - name: Run Pipeline Jar Flink Runner diff --git a/.github/workflows/job-postcommit-portablejar-spark.yml b/.github/workflows/job-postcommit-portablejar-spark.yml index b45fb22e07c3..1dee9fffc7cb 100644 --- a/.github/workflows/job-postcommit-portablejar-spark.yml +++ b/.github/workflows/job-postcommit-portablejar-spark.yml @@ -36,7 +36,6 @@ jobs: uses: actions/checkout@v3 with: persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action - name: Run Pipeline Jar Spark Runner