diff --git a/.github/workflows/job-postcommit-portablejar-flink.yml b/.github/workflows/job-postcommit-portablejar-flink.yml new file mode 100644 index 000000000000..7025982882da --- /dev/null +++ b/.github/workflows/job-postcommit-portablejar-flink.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: PortableJar Flink PostCommit +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] +permissions: read-all + +jobs: + portablejar_flink: + name: Run PortableJar_Flink PostCommit + runs-on: [self-hosted, ubuntu-20.04] + timeout-minutes: 120 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run Pipeline Jar Flink Runner + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner \ No newline at end of file diff --git a/.github/workflows/job-postcommit-portablejar-spark.yml b/.github/workflows/job-postcommit-portablejar-spark.yml new file mode 100644 index 000000000000..1dee9fffc7cb --- /dev/null +++ b/.github/workflows/job-postcommit-portablejar-spark.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: PortableJar Spark PostCommit +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] +permissions: read-all + +jobs: + portablejar_spark: + name: Run PortableJar Spark PostCommit + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run Pipeline Jar Spark Runner + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:portable:py37:testPipelineJarSparkRunner \ No newline at end of file diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 08b9665a92ba..ac017d7f7c4d 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -110,8 +110,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Java_VR_Dataflow_V2 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2_PR/) | `Run Java Dataflow V2 ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2/) | | beam_PostCommit_Java_VR_Dataflow_V2_Streaming | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2_Streaming/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2_Streaming_PR/) | `Run Java Dataflow V2 ValidatesRunner Streaming` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2_Streaming/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_VR_Dataflow_V2_Streaming/) | | beam_PostCommit_Javadoc | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Javadoc/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Javadoc_PR/) | `Run Javadoc PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Javadoc/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Javadoc) | -| beam_PostCommit_PortableJar_Flink | [cron](https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Flink/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Flink_PR/) | `Run PortableJar_Flink PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Flink/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Flink) | -| beam_PostCommit_PortableJar_Spark | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark_PR/) | `Run PortableJar_Spark PostCommit` | [![Build Status](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark/badge/icon)](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_PortableJar_Spark/) | | beam_PostCommit_Py_VR_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_PR/) | `Run Python Dataflow ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow) | | beam_PostCommit_Py_VR_Dataflow_V2 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2_PR/) | `Run Python Dataflow V2 ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2) | | beam_PostCommit_Py_ValCont | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_PR/) | `Run Python Dataflow ValidatesContainer` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont) | diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy deleted file mode 100644 index 1332b61ccb04..000000000000 --- a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// Tests creation and execution of portable pipeline Jars on the Flink runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Flink', - 'Run PortableJar_Flink PostCommit', 'Flink Portable Jar Tests', this) { - description('Tests creation and execution of portable pipeline Jars on the Flink runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120) - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy deleted file mode 100644 index 93e58af8979a..000000000000 --- a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -// Tests creation and execution of portable pipeline Jars on the Spark runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Spark', - 'Run PortableJar_Spark PostCommit', 'Spark Portable Jar Tests', this) { - description('Tests creation and execution of portable pipeline Jars on the Spark runner.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - // Gradle goals for this job. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:portable:py37:testPipelineJarSparkRunner') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/CI.md b/CI.md index ee911584f7f0..ec6b7073f5e3 100644 --- a/CI.md +++ b/CI.md @@ -140,7 +140,8 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog ### PostCommit Workflows | Workflow | Description | Requires GCP Credentials | |------------------------------------------------------------------------------------|-------------------------|--------------------------| -| [job-postcommit-placeholder.yml](.github/workflows/job-postcommit-placeholder.yml) | Description placeholder | Yes/No | +| [job-postcommit-java-portablejar-spark.yml](.github/workflows/job-postcommit-java-portablejar-spark.yml) | Tests creation and execution of portable pipeline Jars on the Spark runner | Yes/No | +| [job-postcommit-java-portablejar-flink.yml](.github/workflows/job-postcommit-java-portablejar-flink.yml) | Tests creation and execution of portable pipeline Jars on the Flink runner | Yes/No | ### GitHub Action Tips diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index ee816a944e5a..70004e4d386b 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -182,7 +182,7 @@ private List gcsCredentialArgs() { // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually. if (Files.exists(Paths.get(localGcloudConfig))) { return ImmutableList.of( - "--mount", + "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); } else { return ImmutableList.of();