GA Migration Python PreCommit Sharding

.github/actions/gradle-command-self-hosted-action/action.yml

@@ -44,6 +44,8 @@ runs:
         # Removing settings.xml is a workaround to avoid a decryption issue
         # of Beam's gradle-command-action plugin and github's provided
         # maven settings.xml file
-        rm ~/.m2/settings.xml
+        if test -f "~/.m2/settings.xml"; then
+          rm ~/.m2/settings.xml
+        fi
         ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} ${{ inputs.arguments }} \
         ${{ inputs.default-arguments }}

.github/actions/setup-self-hosted-action/action.yml

@@ -30,6 +30,10 @@ inputs:
     required: false
     description: 'Set as false if does not require py39 setup'
     default: 'true'
+  requires-py-310:
+    required: false
+    description: 'Set as false if does not require py39 setup'
+    default: 'true'
   requires-java-8:
     required: false
     description: 'Set as false if does not require java-8 setup'
@@ -57,6 +61,11 @@ runs:
       uses: actions/setup-python@v4
       with:
         python-version: "3.9"
+    - name: Install python 3.10
+      if: ${{ inputs.requires-py-310 == 'true'  }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
     - name: Set Java Version
       if: ${{ inputs.requires-java-8 == 'true'  }}
       uses: actions/setup-java@v3

.github/workflows/job-precommit-python-apache-beam.yml

@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PreCommit Python Apache Beam
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+  pull_request_target:
+    branches: [ 'master', 'release-*']
+    tags: 'v*'
+    paths:
+      - 'sdks/python/apache_beam/**'
+permissions: read-all
+
+jobs:
+  set-properties:
+    runs-on: [self-hosted, ubuntu-20.04]
+    outputs:
+      properties: ${{ steps.test-properties.outputs.properties }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - id: test-properties
+        uses: ./.github/actions/setup-default-test-properties
+
+  apache-beam:
+    needs: set-properties
+    name: Run Apache Beam Tests
+    runs-on: [self-hosted, ubuntu-20.04]
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}}
+        tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - name: Set python version
+        run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+        with:
+          requires-go-18: false
+      - name: run apache_beam/pipeline_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/pipeline_test.py"
+      - name: run apache_beam/pvalue_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/pvalue_test.py"

.github/workflows/job-precommit-python-coders.yml

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PreCommit Python Coders
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+  pull_request_target:
+    branches: [ 'master', 'release-*']
+    tags: 'v*'
+    paths:
+      - 'sdks/python/apache_beam/coders/**'
+permissions: read-all
+
+jobs:
+  set-properties:
+    runs-on: [self-hosted, ubuntu-20.04]
+    outputs:
+      properties: ${{ steps.test-properties.outputs.properties }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - id: test-properties
+        uses: ./.github/actions/setup-default-test-properties
+
+  coders:
+    needs: set-properties
+    name: Run Coders Tests
+    runs-on: [ self-hosted, ubuntu-20.04 ]
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}}
+        tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - name: Set python version
+        run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+        with:
+          requires-go-18: false
+      - name: run apache_beam/coders
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/coders/"

.github/workflows/job-precommit-python-internal.yml

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PreCommit Python Internal
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+  pull_request_target:
+    branches: [ 'master', 'release-*']
+    tags: 'v*'
+    paths:
+      - 'sdks/python/apache_beam/internal/**'
+permissions: read-all
+
+jobs:
+  set-properties:
+    runs-on: [self-hosted, ubuntu-20.04]
+    outputs:
+      properties: ${{ steps.test-properties.outputs.properties }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - id: test-properties
+        uses: ./.github/actions/setup-default-test-properties
+
+  internal:
+    needs: set-properties
+    name: Run Internal Tests
+    runs-on: [self-hosted, ubuntu-20.04]
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}}
+        tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - name: Set python version
+        run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+        with:
+          requires-go-18: false
+      - name: run apache_beam/internal
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/internal/"

.github/workflows/job-precommit-python-io.yml

@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PreCommit Python IO
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+  pull_request_target:
+    branches: [ 'master', 'release-*']
+    tags: 'v*'
+    paths:
+      - 'sdks/python/apache_beam/io/concat_source_test.py'
+      - 'sdks/python/apache_beam/io/external/generate_sequence_test.py'
+      - 'sdks/python/apache_beam/io/hadoopfilesystem_test.py'
+      - 'sdks/python/apache_beam/io/iobase_test.py'
+      - 'sdks/python/apache_beam/io/range_trackers_test.py'
+      - 'sdks/python/apache_beam/io/restriction_trackers_test.py'
+      - 'sdks/python/apache_beam/io/source_test_utils_test.py'
+      - 'sdks/python/apache_beam/io/sources_test.py'
+      - 'sdks/python/apache_beam/io/utils_test.py'
+      - 'sdks/python/apache_beam/io/watermark_estimators_test.py'
+permissions: read-all
+
+jobs:
+  set-properties:
+    runs-on: [self-hosted, ubuntu-20.04]
+    outputs:
+      properties: ${{ steps.test-properties.outputs.properties }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - id: test-properties
+        uses: ./.github/actions/setup-default-test-properties
+
+  io:
+    needs: set-properties
+    name: Run IO Tests
+    runs-on: [self-hosted, ubuntu-20.04]
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}}
+        tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+      - name: Set python version
+        run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+        with:
+          requires-go-18: false
+      - name: run apache_beam/io/contact_source_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/io/concat_source_test.py"
+      - name: run apache_beam/io/external/generate_sequence_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/io/external/generate_sequence_test.py"
+      - name: run apache_beam/io/hadoopfilesystem_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/io/hadoopfilesystem_test.py"
+      - name: run apache_beam/io/iobase_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/io/iobase_test.py"
+      - name: run apache_beam/io/range_trackers_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/io/range_trackers_test.py"
+      - name: run apache_beam/io/restriction_trackers_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/io/restriction_trackers_test.py"
+      - name: run apache_beam/io/source_test_utils_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/io/source_test_utils_test.py"
+      - name: run apache_beam/io/sources_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/io/sources_test.py"
+      - name: run apache_beam/io/utils_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/io/utils_test.py"
+      - name: run apache_beam/io/watermark_estimators_test.py
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}}
+          arguments: "-Pposargs=apache_beam/io/watermark_estimators_test.py"
+