GA migration and sharding Python Postcommit ML

.github/workflows/job-postcommit-python-ml.yml

@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This workflow will run the tests of python ML with DataflowRunner.
+
+name: PostCommit Python ML
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*']
+    tags: ['v*']
+permissions: read-all
+
+jobs:
+  set-properties:
+    runs-on: self-hosted
+    outputs:
+      properties: ${{ steps.test-properties.outputs.properties }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+      - id: test-properties
+        uses: ./.github/actions/setup-default-test-properties
+
+  python_ml:
+    needs: set-properties
+    name: Python ML
+    runs-on: [self-hosted, ubuntu-20.04]
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+      - name: Setup service account
+        uses: ./.github/actions/activate-service-account
+        with:
+          GCP_SERVICE_ACCOUNT: ${{secrets.GCP_SELF_HOSTED_SA_KEY}}
+      - name: Set python version
+        run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+        with:
+          requires-go-18: false
+      - name: Run :sdks:python:test-suites:dataflow:py${{env.PYTHON_VERSION}}:postCommitITByTestDirectory
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:dataflow:py${{env.PYTHON_VERSION}}:postCommitITByTestDirectory
+          arguments: "-PtestDir=apache_beam/ml"
+      - name: Run :sdks:python:test-suites:direct:py${{env.PYTHON_VERSION}}:inferencePostCommitIT
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:direct:py${{env.PYTHON_VERSION}}:inferencePostCommitIT

CI.md

@@ -172,9 +172,9 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog
 
 ### PostCommit Workflows
 
-| Workflow                                                                           | Description             | Requires GCP Credentials |
-|------------------------------------------------------------------------------------|-------------------------|--------------------------|
-| [job-postcommit-placeholder.yml](.github/workflows/job-postcommit-placeholder.yml) | Description placeholder | Yes/No                   |
+| Workflow                                                                       | Description                             | Requires GCP Credentials |
+|--------------------------------------------------------------------------------|-----------------------------------------|--------------------------|
+| [job-postcommit-python-ml.yml](.github/workflows/job-postcommit-python-ml.yml) | Runs Python ML Integration Tests | Yes                      |
 
 ### GitHub Action Tips
 

sdks/python/test-suites/dataflow/common.gradle

@@ -95,6 +95,31 @@ task preCommitIT_V2{
   dependsOn preCommitIT_streaming_V2
 }
 
+task postCommitITByTestDirectory {
+  def testDir =  project.hasProperty('testDir') ? project.testDir : ''
+  dependsOn 'installGcpTest'
+  dependsOn ':sdks:python:sdist'
+  dependsOn ':runners:google-cloud-dataflow-java:worker:shadowJar'
+
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+  doLast {
+    def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"]
+    def argMap = [
+            "test_opts": testOpts + [testDir],
+            "sdk_location": files(configurations.distTarBall.files).singleFile,
+            "worker_jar": dataflowWorkerJar,
+            "suite": "postCommitIT-df${pythonVersionSuffix}",
+            "collect": "it_postcommit"
+    ]
+    def cmdArgs = mapToArgString(argMap)
+    exec {
+      executable 'sh'
+      args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
+    }
+  }
+}
+
 task postCommitIT {
   dependsOn 'installGcpTest'
   dependsOn ':sdks:python:sdist'