From 1f5dbc633dd6664fe84853577373529e1930f90e Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Mon, 19 Sep 2022 13:28:12 -0500 Subject: [PATCH 1/2] Migrated github action --- .../workflows/job-postcommit-xvr-direct.yml | 75 +++++++++++++++++++ .../workflows/job-postcommit-xvr-flink.yml | 72 ++++++++++++++++++ ...ob-postcommit-xvr-gousingjava-dataflow.yml | 55 ++++++++++++++ ...ostcommit-xvr-javausingpython-dataflow.yml | 72 ++++++++++++++++++ ...ostcommit-xvr-pythonusingjava-dataflow.yml | 72 ++++++++++++++++++ .../environment/DockerEnvironmentFactory.java | 3 +- 6 files changed, 347 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/job-postcommit-xvr-direct.yml create mode 100644 .github/workflows/job-postcommit-xvr-flink.yml create mode 100644 .github/workflows/job-postcommit-xvr-gousingjava-dataflow.yml create mode 100644 .github/workflows/job-postcommit-xvr-javausingpython-dataflow.yml create mode 100644 .github/workflows/job-postcommit-xvr-pythonusingjava-dataflow.yml diff --git a/.github/workflows/job-postcommit-xvr-direct.yml b/.github/workflows/job-postcommit-xvr-direct.yml new file mode 100644 index 000000000000..d9be8dcaf5ad --- /dev/null +++ b/.github/workflows/job-postcommit-xvr-direct.yml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow will triage pull requests and apply a label based on the +# paths that are modified in the pull request. + +# Runs the CrossLanguageValidatesRunner suite on the Direct runner + +name: PostCommit XVR Direct + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-xvr-migration'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + xvr-direct: + needs: set-properties + name: PostCommit XVR Direct + runs-on: [self-hosted, ubuntu-20.04] + strategy: + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + + - name: Run :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner + arguments: "-PpythonVersion=${{ matrix.version }}" + + + diff --git a/.github/workflows/job-postcommit-xvr-flink.yml b/.github/workflows/job-postcommit-xvr-flink.yml new file mode 100644 index 000000000000..884d74c7ddcd --- /dev/null +++ b/.github/workflows/job-postcommit-xvr-flink.yml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow will triage pull requests and apply a label based on the +# paths that are modified in the pull request. + +# Runs the CrossLanguageValidatesRunner suite on the Flink runner + +name: PostCommit XVR Flink + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-xvr-migration'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + xvr-flink: + needs: set-properties + name: PostCommit XVR Flink + runs-on: [self-hosted, ubuntu-20.04] + strategy: + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + + - name: Run :runners:flink:1.13:job-server:validatesCrossLanguageRunner + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:1.13:job-server:validatesCrossLanguageRunner + arguments: "-PpythonVersion=${{ matrix.version }}" diff --git a/.github/workflows/job-postcommit-xvr-gousingjava-dataflow.yml b/.github/workflows/job-postcommit-xvr-gousingjava-dataflow.yml new file mode 100644 index 000000000000..4cace80e33b4 --- /dev/null +++ b/.github/workflows/job-postcommit-xvr-gousingjava-dataflow.yml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow will triage pull requests and apply a label based on the +# paths that are modified in the pull request. + +# Runs the CrossLanguageValidatesRunner GoUsingJava suite on the Dataflow runner + +name: PostCommit XVR Go Using Java Dataflow + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-xvr-migration'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' +permissions: read-all + +jobs: + xvr-dataflow: + name: PostCommit XVR Go Using Java Dataflow + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + + - name: Run :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava + uses: ./.github/actions/gradle-command-self-hosted-action + env: + USER: beam-sdk + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava \ No newline at end of file diff --git a/.github/workflows/job-postcommit-xvr-javausingpython-dataflow.yml b/.github/workflows/job-postcommit-xvr-javausingpython-dataflow.yml new file mode 100644 index 000000000000..00e49b23e005 --- /dev/null +++ b/.github/workflows/job-postcommit-xvr-javausingpython-dataflow.yml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow will triage pull requests and apply a label based on the +# paths that are modified in the pull request. + +# Runs the CrossLanguageValidatesRunner JavaUsingPython suite on the Dataflow runner + +name: PostCommit XVR Java Using Python Dataflow + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-xvr-migration'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + xvr-dataflow: + needs: set-properties + name: PostCommit XVR Java Using Python Dataflow + runs-on: [self-hosted, ubuntu-20.04] + strategy: + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + + - name: Run :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerJavaUsingPython + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerJavaUsingPython + arguments: "-PpythonVersion=${{ matrix.version }}" \ No newline at end of file diff --git a/.github/workflows/job-postcommit-xvr-pythonusingjava-dataflow.yml b/.github/workflows/job-postcommit-xvr-pythonusingjava-dataflow.yml new file mode 100644 index 000000000000..41d429752c56 --- /dev/null +++ b/.github/workflows/job-postcommit-xvr-pythonusingjava-dataflow.yml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow will triage pull requests and apply a label based on the +# paths that are modified in the pull request. + +# Runs the CrossLanguageValidatesRunner PythonUsingJava suite on the Dataflow runner + +name: PostCommit XVR Python Using Java Dataflow + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-xvr-migration'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + xvr-dataflow: + needs: set-properties + name: PostCommit XVR Python Using Java Dataflow + runs-on: [self-hosted, ubuntu-20.04] + strategy: + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + + - name: Run :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingJava + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingJava + arguments: "-PpythonVersion=${{ matrix.version }}" \ No newline at end of file diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index ee816a944e5a..edead73227d7 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -182,8 +182,7 @@ private List gcsCredentialArgs() { // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually. if (Files.exists(Paths.get(localGcloudConfig))) { return ImmutableList.of( - "--mount", - String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); + "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); } else { return ImmutableList.of(); } From 39cd0e9881d4f48982e355b873a4e679d1d9e831 Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Thu, 29 Sep 2022 16:01:05 -0500 Subject: [PATCH 2/2] Updated CI.md --- .../workflows/job-postcommit-xvr-direct.yml | 10 ++-- .../workflows/job-postcommit-xvr-flink.yml | 7 ++- ...ob-postcommit-xvr-gousingjava-dataflow.yml | 3 +- ...ostcommit-xvr-javausingpython-dataflow.yml | 5 +- ...ostcommit-xvr-pythonusingjava-dataflow.yml | 5 +- ...CrossLanguageValidatesRunner_Direct.groovy | 49 ------------------- ...atesRunner_JavaUsingPython_Dataflow.groovy | 49 ------------------- ...atesRunner_PythonUsingJava_Dataflow.groovy | 49 ------------------- CI.md | 10 ++-- 9 files changed, 18 insertions(+), 169 deletions(-) delete mode 100644 .test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy delete mode 100644 .test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_JavaUsingPython_Dataflow.groovy delete mode 100644 .test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJava_Dataflow.groovy diff --git a/.github/workflows/job-postcommit-xvr-direct.yml b/.github/workflows/job-postcommit-xvr-direct.yml index d9be8dcaf5ad..8c9b57670bcc 100644 --- a/.github/workflows/job-postcommit-xvr-direct.yml +++ b/.github/workflows/job-postcommit-xvr-direct.yml @@ -25,7 +25,7 @@ on: schedule: - cron: '0 */6 * * *' push: - branches: ['master', 'release-*', 'postcommit-xvr-migration'] + branches: ['master', 'release-*'] tags: 'v*' pull_request_target: branches: [ 'master', 'release-*' ] @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -52,6 +51,7 @@ jobs: name: PostCommit XVR Direct runs-on: [self-hosted, ubuntu-20.04] strategy: + fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} steps: @@ -60,7 +60,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action @@ -69,7 +68,4 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner - arguments: "-PpythonVersion=${{ matrix.version }}" - - - + arguments: "-PpythonVersion=${{ matrix.version }}" \ No newline at end of file diff --git a/.github/workflows/job-postcommit-xvr-flink.yml b/.github/workflows/job-postcommit-xvr-flink.yml index 884d74c7ddcd..1a17c426794f 100644 --- a/.github/workflows/job-postcommit-xvr-flink.yml +++ b/.github/workflows/job-postcommit-xvr-flink.yml @@ -25,7 +25,7 @@ on: schedule: - cron: '0 */6 * * *' push: - branches: ['master', 'release-*', 'postcommit-xvr-migration'] + branches: ['master', 'release-*'] tags: 'v*' pull_request_target: branches: [ 'master', 'release-*' ] @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -52,6 +51,7 @@ jobs: name: PostCommit XVR Flink runs-on: [self-hosted, ubuntu-20.04] strategy: + fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} steps: @@ -60,7 +60,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action @@ -69,4 +68,4 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:flink:1.13:job-server:validatesCrossLanguageRunner - arguments: "-PpythonVersion=${{ matrix.version }}" + arguments: "-PpythonVersion=${{ matrix.version }}" \ No newline at end of file diff --git a/.github/workflows/job-postcommit-xvr-gousingjava-dataflow.yml b/.github/workflows/job-postcommit-xvr-gousingjava-dataflow.yml index 4cace80e33b4..e1075d51f9e1 100644 --- a/.github/workflows/job-postcommit-xvr-gousingjava-dataflow.yml +++ b/.github/workflows/job-postcommit-xvr-gousingjava-dataflow.yml @@ -25,7 +25,7 @@ on: schedule: - cron: '0 */6 * * *' push: - branches: ['master', 'release-*', 'postcommit-xvr-migration'] + branches: ['master', 'release-*'] tags: 'v*' pull_request_target: branches: [ 'master', 'release-*' ] @@ -42,7 +42,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action diff --git a/.github/workflows/job-postcommit-xvr-javausingpython-dataflow.yml b/.github/workflows/job-postcommit-xvr-javausingpython-dataflow.yml index 00e49b23e005..f1747f969c05 100644 --- a/.github/workflows/job-postcommit-xvr-javausingpython-dataflow.yml +++ b/.github/workflows/job-postcommit-xvr-javausingpython-dataflow.yml @@ -25,7 +25,7 @@ on: schedule: - cron: '0 */6 * * *' push: - branches: ['master', 'release-*', 'postcommit-xvr-migration'] + branches: ['master', 'release-*'] tags: 'v*' pull_request_target: branches: [ 'master', 'release-*' ] @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -52,6 +51,7 @@ jobs: name: PostCommit XVR Java Using Python Dataflow runs-on: [self-hosted, ubuntu-20.04] strategy: + fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} steps: @@ -60,7 +60,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action diff --git a/.github/workflows/job-postcommit-xvr-pythonusingjava-dataflow.yml b/.github/workflows/job-postcommit-xvr-pythonusingjava-dataflow.yml index 41d429752c56..bb07078edf99 100644 --- a/.github/workflows/job-postcommit-xvr-pythonusingjava-dataflow.yml +++ b/.github/workflows/job-postcommit-xvr-pythonusingjava-dataflow.yml @@ -25,7 +25,7 @@ on: schedule: - cron: '0 */6 * * *' push: - branches: ['master', 'release-*', 'postcommit-xvr-migration'] + branches: ['master', 'release-*'] tags: 'v*' pull_request_target: branches: [ 'master', 'release-*' ] @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -52,6 +51,7 @@ jobs: name: PostCommit XVR Python Using Java Dataflow runs-on: [self-hosted, ubuntu-20.04] strategy: + fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} steps: @@ -60,7 +60,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy deleted file mode 100644 index d8362c1e1e53..000000000000 --- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS - -// This job runs the suite of ValidatesRunner tests against the Direct runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Direct', - 'Run XVR_Direct PostCommit', 'Direct CrossLanguageValidatesRunner Tests', this) { - description('Runs the CrossLanguageValidatesRunner suite on the Direct runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - // Gradle goals for this job. - steps { - CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion -> - shell("echo \"*** RUN CROSS-LANGUAGE DIRECT USING PYTHON ${pythonVersion} ***\"") - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner') - commonJobProperties.setGradleSwitches(delegate) - switches("-PpythonVersion=${pythonVersion}") - } - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_JavaUsingPython_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_JavaUsingPython_Dataflow.groovy deleted file mode 100644 index 764bf697cbee..000000000000 --- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_JavaUsingPython_Dataflow.groovy +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS - -// This job runs the suite of ValidatesRunner tests against the Dataflow runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_JavaUsingPython_Dataflow', - 'Run XVR_JavaUsingPython_Dataflow PostCommit', 'Dataflow CrossLanguageValidatesRunner JavaUsingPython Tests', this) { - description('Runs the CrossLanguageValidatesRunner JavaUsingPython suite on the Dataflow runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } - - // Gradle goals for this job. - steps { - CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion -> - shell("echo \"*** RUN CROSS-LANGUAGE DATAFLOW JAVA WITH PYTHON EXTERNAL TRANSFORMS USING PYTHON ${pythonVersion} ***\"") - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerJavaUsingPython') - commonJobProperties.setGradleSwitches(delegate) - switches("-PpythonVersion=${pythonVersion}") - } - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJava_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJava_Dataflow.groovy deleted file mode 100644 index 03745685a299..000000000000 --- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJava_Dataflow.groovy +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS - -// This job runs the suite of ValidatesRunner tests against the Dataflow runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_PythonUsingJava_Dataflow', - 'Run XVR_PythonUsingJava_Dataflow PostCommit', 'Dataflow CrossLanguageValidatesRunner PythonUsingJava Tests', this) { - description('Runs the CrossLanguageValidatesRunner PythonUsingJava suite on the Dataflow runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Publish all test results to Jenkins - publishers { - archiveJunit('**/pytest*.xml') - } - - // Gradle goals for this job. - steps { - CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion -> - shell("echo \"*** RUN CROSS-LANGUAGE DATAFLOW PYTHON WITH JAVA EXTERNAL TRANSFORMS USING PYTHON ${pythonVersion} ***\"") - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingJava') - commonJobProperties.setGradleSwitches(delegate) - switches("-PpythonVersion=${pythonVersion}") - } - } - } - } diff --git a/CI.md b/CI.md index ee911584f7f0..d07a62f5d0a7 100644 --- a/CI.md +++ b/CI.md @@ -138,9 +138,13 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | ### PostCommit Workflows -| Workflow | Description | Requires GCP Credentials | -|------------------------------------------------------------------------------------|-------------------------|--------------------------| -| [job-postcommit-placeholder.yml](.github/workflows/job-postcommit-placeholder.yml) | Description placeholder | Yes/No | +| Workflow | Description | Requires GCP Credentials | +|----------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------|--------------------------| +| [job-postcommit-xvr-direct.yml](.github/workflows/job-postcommit-xvr-direct.yml) | Runs the CrossLanguageValidatesRunner suite on the Direct runner | No | +| [job-postcommit-xvr-flink.yml](.github/workflows/job-postcommit-xvr-flink.yml) | Runs the CrossLanguageValidatesRunner suite on the Flink runner | No | +| [job-postcommit-xvr-gousingjava-dataflow.yml](.github/workflows/job-postcommit-xvr-gousingjava-dataflow.yml) | Runs the CrossLanguageValidatesRunner GoUsingJava suite on the Dataflow runner | No | +| [job-postcommit-xvr-javausingpython-dataflow.yml](.github/workflows/job-postcommit-xvr-javausingpython-dataflow.yml) | Runs the CrossLanguageValidatesRunner JavaUsingPython suite on the Dataflow runner | No | +| [job-postcommit-xvr-pythonusingjava-dataflow.yml](.github/workflows/job-postcommit-xvr-pythonusingjava-dataflow.yml) | Runs the CrossLanguageValidatesRunner PythonUsingJava suite on the Dataflow runner | No | ### GitHub Action Tips