From df83bb9369f5f350119cb81d0fecb90f2bd38936 Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Mon, 19 Sep 2022 13:32:25 -0500 Subject: [PATCH 1/2] Migrated github action --- .../job-postcommit-python-vr-flink.yml | 65 ++++++++++++++++++ .../job-postcommit-python-vr-samza.yml | 64 ++++++++++++++++++ .../job-postcommit-python-vr-spark.yml | 66 +++++++++++++++++++ .../environment/DockerEnvironmentFactory.java | 3 +- 4 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/job-postcommit-python-vr-flink.yml create mode 100644 .github/workflows/job-postcommit-python-vr-samza.yml create mode 100644 .github/workflows/job-postcommit-python-vr-spark.yml diff --git a/.github/workflows/job-postcommit-python-vr-flink.yml b/.github/workflows/job-postcommit-python-vr-flink.yml new file mode 100644 index 000000000000..f9ce79e94007 --- /dev/null +++ b/.github/workflows/job-postcommit-python-vr-flink.yml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Runs the Python ValidatesRunner suite on the Flink runner + +name: PostCommit Python VR Flink + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-vr-migration'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' +permissions: read-all + +jobs: + set-properties: + runs-on: [ self-hosted, ubuntu-20.04 ] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python-vr-flink: + needs: set-properties + name: PostCommit Python VR Flink + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:python:test-suites:portable:flinkValidatesRunner + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:flinkValidatesRunner diff --git a/.github/workflows/job-postcommit-python-vr-samza.yml b/.github/workflows/job-postcommit-python-vr-samza.yml new file mode 100644 index 000000000000..8325b110e639 --- /dev/null +++ b/.github/workflows/job-postcommit-python-vr-samza.yml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Runs the Python ValidatesRunner suite on the Samza runner + +name: PostCommit Python VR Samza + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-vr-migration'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' +permissions: read-all + +jobs: + set-properties: + runs-on: [ self-hosted, ubuntu-20.04 ] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python-vr-samza: + needs: set-properties + name: PostCommit Python VR Samza + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:python:test-suites:portable:samzaValidatesRunner + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:samzaValidatesRunner diff --git a/.github/workflows/job-postcommit-python-vr-spark.yml b/.github/workflows/job-postcommit-python-vr-spark.yml new file mode 100644 index 000000000000..7bdde03d8112 --- /dev/null +++ b/.github/workflows/job-postcommit-python-vr-spark.yml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Runs the Python ValidatesRunner suite on the Spark runner + +name: PostCommit Python VR Spark + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-vr-migration'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' +permissions: read-all + +jobs: + set-properties: + runs-on: [ self-hosted, ubuntu-20.04 ] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python-vr-spark: + needs: set-properties + name: PostCommit Python VR Spark + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :pythonSparkPostCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :pythonSparkPostCommit + + diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index ee816a944e5a..edead73227d7 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -182,8 +182,7 @@ private List gcsCredentialArgs() { // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually. if (Files.exists(Paths.get(localGcloudConfig))) { return ImmutableList.of( - "--mount", - String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); + "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); } else { return ImmutableList.of(); } From f4fc7c9e4b2c6815f6c72e641c1f0bbfd13f3728 Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Thu, 29 Sep 2022 16:10:25 -0500 Subject: [PATCH 2/2] Updated CI.md --- .../job-postcommit-python-vr-flink.yml | 4 +- .../job-postcommit-python-vr-samza.yml | 4 +- .../job-postcommit-python-vr-spark.yml | 4 +- ...Commit_Python_ValidatesRunner_Flink.groovy | 38 ---------------- ...Commit_Python_ValidatesRunner_Samza.groovy | 43 ------------------- ...Commit_Python_ValidatesRunner_Spark.groovy | 43 ------------------- CI.md | 8 ++-- 7 files changed, 8 insertions(+), 136 deletions(-) delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Flink.groovy delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Samza.groovy delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy diff --git a/.github/workflows/job-postcommit-python-vr-flink.yml b/.github/workflows/job-postcommit-python-vr-flink.yml index f9ce79e94007..82896ce758fa 100644 --- a/.github/workflows/job-postcommit-python-vr-flink.yml +++ b/.github/workflows/job-postcommit-python-vr-flink.yml @@ -24,7 +24,7 @@ on: schedule: - cron: '0 */6 * * *' push: - branches: ['master', 'release-*', 'postcommit-vr-migration'] + branches: ['master', 'release-*'] tags: 'v*' pull_request_target: branches: [ 'master', 'release-*'] @@ -42,7 +42,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -56,7 +55,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action - name: Run :sdks:python:test-suites:portable:flinkValidatesRunner diff --git a/.github/workflows/job-postcommit-python-vr-samza.yml b/.github/workflows/job-postcommit-python-vr-samza.yml index 8325b110e639..62dd43d43f88 100644 --- a/.github/workflows/job-postcommit-python-vr-samza.yml +++ b/.github/workflows/job-postcommit-python-vr-samza.yml @@ -23,7 +23,7 @@ on: schedule: - cron: '0 */6 * * *' push: - branches: ['master', 'release-*', 'postcommit-vr-migration'] + branches: ['master', 'release-*'] tags: 'v*' pull_request_target: branches: [ 'master', 'release-*'] @@ -41,7 +41,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -55,7 +54,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action - name: Run :sdks:python:test-suites:portable:samzaValidatesRunner diff --git a/.github/workflows/job-postcommit-python-vr-spark.yml b/.github/workflows/job-postcommit-python-vr-spark.yml index 7bdde03d8112..9c6d18279f6f 100644 --- a/.github/workflows/job-postcommit-python-vr-spark.yml +++ b/.github/workflows/job-postcommit-python-vr-spark.yml @@ -23,7 +23,7 @@ on: schedule: - cron: '0 */6 * * *' push: - branches: ['master', 'release-*', 'postcommit-vr-migration'] + branches: ['master', 'release-*'] tags: 'v*' pull_request_target: branches: [ 'master', 'release-*'] @@ -41,7 +41,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -55,7 +54,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action - name: Run :pythonSparkPostCommit diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Flink.groovy deleted file mode 100644 index 5f0aa8f13b9d..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Flink.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the suite of Python ValidatesRunner tests against the Flink runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_VR_Flink', - 'Run Python Flink ValidatesRunner', 'Run Python Flink ValidatesRunner', this) { - description('Runs the Python ValidatesRunner suite on the Flink runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:portable:flinkValidatesRunner') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Samza.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Samza.groovy deleted file mode 100644 index c49e3677ab8a..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Samza.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the suite of Python ValidatesRunner tests against the Samza runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_VR_Samza', - 'Run Python Samza ValidatesRunner', 'Python Samza ValidatesRunner Tests', this) { - description('Runs the Python ValidatesRunner suite on the Samza runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:portable:samzaValidatesRunner') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy deleted file mode 100644 index ce71871da7eb..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// This job runs the suite of Python ValidatesRunner tests against the Spark runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_VR_Spark', - 'Run Python Spark ValidatesRunner', 'Python Spark ValidatesRunner Tests', this) { - description('Runs the Python ValidatesRunner suite on the Spark runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Publish all test results to Jenkins. - publishers { - archiveJunit('**/pytest*.xml') - } - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':pythonSparkPostCommit') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/CI.md b/CI.md index ee911584f7f0..d77b38140bac 100644 --- a/CI.md +++ b/CI.md @@ -138,9 +138,11 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | ### PostCommit Workflows -| Workflow | Description | Requires GCP Credentials | -|------------------------------------------------------------------------------------|-------------------------|--------------------------| -| [job-postcommit-placeholder.yml](.github/workflows/job-postcommit-placeholder.yml) | Description placeholder | Yes/No | +| Workflow | Description | Requires GCP Credentials | +|--------------------------------------------------------------------------------------------|-----------------------------------------------------------|--------------------------| +| [job-postcommit-python-vr-flink.yml](.github/workflows/job-postcommit-python-vr-flink.yml) | Runs the Python ValidatesRunner suite on the Flink runner | No | +| [job-postcommit-python-vr-samza.yml](.github/workflows/job-postcommit-python-vr-samza.yml) | Runs the Python ValidatesRunner suite on the Samza runner | No | +| [job-postcommit-python-vr-spark.yml](.github/workflows/job-postcommit-python-vr-spark.yml) | Runs the Python ValidatesRunner suite on the Spark runner | No | ### GitHub Action Tips