GA Migration Postcommit Go VR Flink, Samza, and Spark

.github/workflows/job-postcommit-python-vr-flink.yml

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Runs the Python ValidatesRunner suite on the Flink runner
+
+name: PostCommit Python VR Flink
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+  pull_request_target:
+    branches: [ 'master', 'release-*']
+    tags: 'v*'
+permissions: read-all
+
+jobs:
+  set-properties:
+    runs-on: [ self-hosted, ubuntu-20.04 ]
+    outputs:
+      properties: ${{ steps.test-properties.outputs.properties }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - id: test-properties
+        uses: ./.github/actions/setup-default-test-properties
+
+  python-vr-flink:
+    needs: set-properties
+    name: PostCommit Python VR Flink
+    runs-on: [self-hosted, ubuntu-20.04]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+      - name: Run :sdks:python:test-suites:portable:flinkValidatesRunner
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:portable:flinkValidatesRunner

.github/workflows/job-postcommit-python-vr-samza.yml

@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Runs the Python ValidatesRunner suite on the Samza runner
+
+name: PostCommit Python VR Samza
+
+on:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+  pull_request_target:
+    branches: [ 'master', 'release-*']
+    tags: 'v*'
+permissions: read-all
+
+jobs:
+  set-properties:
+    runs-on: [ self-hosted, ubuntu-20.04 ]
+    outputs:
+      properties: ${{ steps.test-properties.outputs.properties }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - id: test-properties
+        uses: ./.github/actions/setup-default-test-properties
+
+  python-vr-samza:
+    needs: set-properties
+    name: PostCommit Python VR Samza
+    runs-on: [self-hosted, ubuntu-20.04]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+      - name: Run :sdks:python:test-suites:portable:samzaValidatesRunner
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:portable:samzaValidatesRunner

.github/workflows/job-postcommit-python-vr-spark.yml

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Runs the Python ValidatesRunner suite on the Spark runner
+
+name: PostCommit Python VR Spark
+
+on:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+  pull_request_target:
+    branches: [ 'master', 'release-*']
+    tags: 'v*'
+permissions: read-all
+
+jobs:
+  set-properties:
+    runs-on: [ self-hosted, ubuntu-20.04 ]
+    outputs:
+      properties: ${{ steps.test-properties.outputs.properties }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - id: test-properties
+        uses: ./.github/actions/setup-default-test-properties
+
+  python-vr-spark:
+    needs: set-properties
+    name: PostCommit Python VR Spark
+    runs-on: [self-hosted, ubuntu-20.04]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+      - name: Run :pythonSparkPostCommit
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :pythonSparkPostCommit
+
+

CI.md

@@ -159,10 +159,12 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog
 | [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No                    |
 
 ### PostCommit Workflows
+| Workflow                                                                                   | Description                                               | Requires GCP Credentials |
+|--------------------------------------------------------------------------------------------|-----------------------------------------------------------|--------------------------|
+| [job-postcommit-python-vr-flink.yml](.github/workflows/job-postcommit-python-vr-flink.yml) | Runs the Python ValidatesRunner suite on the Flink runner | No                       |
+| [job-postcommit-python-vr-samza.yml](.github/workflows/job-postcommit-python-vr-samza.yml) | Runs the Python ValidatesRunner suite on the Samza runner | No                       |
+| [job-postcommit-python-vr-spark.yml](.github/workflows/job-postcommit-python-vr-spark.yml) | Runs the Python ValidatesRunner suite on the Spark runner | No                       |
 
-| Workflow                                                                           | Description             | Requires GCP Credentials |
-|------------------------------------------------------------------------------------|-------------------------|--------------------------|
-| [job-postcommit-placeholder.yml](.github/workflows/job-postcommit-placeholder.yml) | Description placeholder | Yes/No                   |
 
 ### GitHub Action Tips
 

runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java

@@ -182,8 +182,7 @@ private List<String> gcsCredentialArgs() {
     // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually.
     if (Files.exists(Paths.get(localGcloudConfig))) {
       return ImmutableList.of(
-          "--mount",
-          String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig));
+          "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig));
     } else {
       return ImmutableList.of();
     }