From 2bed71bd1aa96861619cc0a78f8a3cdb69e5fd3e Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Fri, 30 Sep 2022 10:14:02 -0500 Subject: [PATCH 1/4] Migrated github action --- .github/workflows/job-precommit-python.yml | 275 +++++++++++++++++++++ CI.md | 6 +- 2 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/job-precommit-python.yml diff --git a/.github/workflows/job-precommit-python.yml b/.github/workflows/job-precommit-python.yml new file mode 100644 index 000000000000..a87550de5e7a --- /dev/null +++ b/.github/workflows/job-precommit-python.yml @@ -0,0 +1,275 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/**' + - 'sdks/python/apache_beam/runners/worker/**' + - 'sdks/python/apache_beam/testing/**' + - 'sdks/python/apache_beam/transforms/**' + - 'sdks/python/apache_beam/typehints/**' + - 'sdks/python/apache_beam/utils/**' + - 'sdks/python/apache_beam/coders/**' + - 'sdks/python/apache_beam/internal/**' + - 'sdks/python/apache_beam/metrics/**' + - 'sdks/python/apache_beam/options/**' + - 'sdks/python/apache_beam/runners/portability/fn_api_runner/**' + - 'sdks/python/apache_beam/runners/portability/stager_test.py' + - 'sdks/python/apache_beam/runners/runner_test.py' + - 'sdks/python/apache_beam/runners/sdf_utils_test.py' + - 'sdks/python/apache_beam/tools/microbenchmarks_test.py' + - 'sdks/python/apache_beam/io/concat_source_test.py' + - 'sdks/python/apache_beam/io/external/generate_sequence_test.py' + - 'sdks/python/apache_beam/io/iobase_test.py' + - 'sdks/python/apache_beam/io/range_trackers_test.py' + - 'sdks/python/apache_beam/io/restriction_trackers_test.py' + - 'sdks/python/apache_beam/io/source_test_utils_test.py' + - 'sdks/python/apache_beam/io/sources_test.py' + - 'sdks/python/apache_beam/io/utils_test.py' + - 'sdks/python/apache_beam/io/watermark_estimators_test.py' + - 'sdks/python/apache_beam/runners/common_test.py' + - 'sdks/python/apache_beam/runners/pipeline_context_test.py' + - 'sdks/python/apache_beam/runners/portability/artifact_service_test.py' + - 'sdks/python/apache_beam/runners/portability/job_server_test.py' + - 'sdks/python/apache_beam/runners/portability/local_job_service_test.py' + - 'sdks/python/apache_beam/runners/portability/portable_runner_test.py' + - 'sdks/python/apache_beam/runners/portability/sdk_container_builder_test.py' + - 'sdks/python/apache_beam/io/hadoopfilesystem_test.py' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + precommit-python: + needs: set-properties + name: PreCommit Python + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + + - name: Run Apache Beam Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/" + + - name: Run Runners Worker Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/worker/" + + - name: Run Testing Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/testing/" + + - name: Run Transform Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/transforms/" + + - name: Run Typehints Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/typehints/" + + - name: Run Utils Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/utils/" + + - name: Run Coders Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/coders/" + + - name: Run Internal Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/internal/" + + - name: Run Metrics Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/metrics/" + + - name: Run Options Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/options/" + + - name: Run Runners Portability FnApiRunner Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/fn_api_runner/" + - name: Run Runners Portability Stager Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/stager_test.py" + - name: Run Runners Runner Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/runner_test.py" + - name: Run Runners SDF Utils Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/sdf_utils_test.py" + + - name: Run Tools Microbenchmarks Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/tools/microbenchmarks_test.py" + + - name: Run IO Concat Source Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/concat_source_test.py" + - name: Run IO External Generate Sequence Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/external/generate_sequence_test.py" + - name: Run IO IOBase Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/iobase_test.py" + - name: Run IO Range Trackers Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/range_trackers_test.py" + - name: Run IO Restrinction Trackers Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/restriction_trackers_test.py" + - name: Run IO Source Test Utils Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/source_test_utils_test.py" + - name: Run IO Sources Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/sources_test.py" + - name: Run IO Utils Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/utils_test.py" + - name: Run IO Watermark Estimators Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/watermark_estimators_test.py" + + - name: Run Runners Common Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/common_test.py" + - name: Run Runners Pipeline Context Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/pipeline_context_test.py" + - name: Run Runners Portability Artifact Service Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/artifact_service_test.py" + + - name: Run Runners Portability Job Server Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/job_server_test.py" + - name: Run Runners Local Job Service Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/local_job_service_test.py" + - name: Run Runners Portability Portable Runner Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/portable_runner_test.py" + - name: Run Runners Portability SDK Container Builder Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/sdk_container_builder_test.py" + + - name: Run IO Hadoop File System Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/hadoopfilesystem_test.py" \ No newline at end of file diff --git a/CI.md b/CI.md index ee911584f7f0..bb4a580fc6ac 100644 --- a/CI.md +++ b/CI.md @@ -133,9 +133,9 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | PreCommit | Yes | Yes | Yes | Yes | ### PreCommit Workflows -| Workflow | Description | Requires GCP Credentials | -|----------------------------------------------------------------------------------|-------------------------|---------------------------| -| [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | +| Workflow | Description | Requires GCP Credentials | +|------------------------------------------------------------------------|---------------------------------------------------|--------------------------| +| [job-precommit-python.yml](.github/workflows/job-precommit-python.yml) | Runs multiple Python test for each of its modules | No | ### PostCommit Workflows | Workflow | Description | Requires GCP Credentials | From 39e0bc4f37612e5a5bdc339ac0ea7be625d06546 Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Mon, 17 Oct 2022 11:46:51 -0500 Subject: [PATCH 2/4] Alphabetically ordered paths --- .github/workflows/job-precommit-python.yml | 26 +++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/job-precommit-python.yml b/.github/workflows/job-precommit-python.yml index a87550de5e7a..5e88f5fb2b13 100644 --- a/.github/workflows/job-precommit-python.yml +++ b/.github/workflows/job-precommit-python.yml @@ -29,22 +29,11 @@ on: tags: 'v*' paths: - 'sdks/python/apache_beam/**' - - 'sdks/python/apache_beam/runners/worker/**' - - 'sdks/python/apache_beam/testing/**' - - 'sdks/python/apache_beam/transforms/**' - - 'sdks/python/apache_beam/typehints/**' - - 'sdks/python/apache_beam/utils/**' - 'sdks/python/apache_beam/coders/**' - 'sdks/python/apache_beam/internal/**' - - 'sdks/python/apache_beam/metrics/**' - - 'sdks/python/apache_beam/options/**' - - 'sdks/python/apache_beam/runners/portability/fn_api_runner/**' - - 'sdks/python/apache_beam/runners/portability/stager_test.py' - - 'sdks/python/apache_beam/runners/runner_test.py' - - 'sdks/python/apache_beam/runners/sdf_utils_test.py' - - 'sdks/python/apache_beam/tools/microbenchmarks_test.py' - 'sdks/python/apache_beam/io/concat_source_test.py' - 'sdks/python/apache_beam/io/external/generate_sequence_test.py' + - 'sdks/python/apache_beam/io/hadoopfilesystem_test.py' - 'sdks/python/apache_beam/io/iobase_test.py' - 'sdks/python/apache_beam/io/range_trackers_test.py' - 'sdks/python/apache_beam/io/restriction_trackers_test.py' @@ -52,14 +41,25 @@ on: - 'sdks/python/apache_beam/io/sources_test.py' - 'sdks/python/apache_beam/io/utils_test.py' - 'sdks/python/apache_beam/io/watermark_estimators_test.py' + - 'sdks/python/apache_beam/metrics/**' + - 'sdks/python/apache_beam/options/**' - 'sdks/python/apache_beam/runners/common_test.py' - 'sdks/python/apache_beam/runners/pipeline_context_test.py' - 'sdks/python/apache_beam/runners/portability/artifact_service_test.py' + - 'sdks/python/apache_beam/runners/portability/fn_api_runner/**' - 'sdks/python/apache_beam/runners/portability/job_server_test.py' - 'sdks/python/apache_beam/runners/portability/local_job_service_test.py' - 'sdks/python/apache_beam/runners/portability/portable_runner_test.py' - 'sdks/python/apache_beam/runners/portability/sdk_container_builder_test.py' - - 'sdks/python/apache_beam/io/hadoopfilesystem_test.py' + - 'sdks/python/apache_beam/runners/portability/stager_test.py' + - 'sdks/python/apache_beam/runners/runner_test.py' + - 'sdks/python/apache_beam/runners/sdf_utils_test.py' + - 'sdks/python/apache_beam/runners/worker/**' + - 'sdks/python/apache_beam/testing/**' + - 'sdks/python/apache_beam/tools/microbenchmarks_test.py' + - 'sdks/python/apache_beam/transforms/**' + - 'sdks/python/apache_beam/typehints/**' + - 'sdks/python/apache_beam/utils/**' permissions: read-all jobs: From 48c43861d2e15ba381216a90ae18092caf0646d5 Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Wed, 30 Nov 2022 10:48:04 -0600 Subject: [PATCH 3/4] Splitted test into jobs --- .github/workflows/job-precommit-python.yml | 377 +++++++++++---------- sdks/python/test-suites/tox/common.gradle | 9 +- 2 files changed, 210 insertions(+), 176 deletions(-) diff --git a/.github/workflows/job-precommit-python.yml b/.github/workflows/job-precommit-python.yml index 5e88f5fb2b13..8ac68edbf0b1 100644 --- a/.github/workflows/job-precommit-python.yml +++ b/.github/workflows/job-precommit-python.yml @@ -22,7 +22,7 @@ on: schedule: - cron: '0 */6 * * *' push: - branches: ['master', 'release-*'] + branches: ['master', 'release-*', 'sharding-precommit-python'] tags: 'v*' pull_request_target: branches: [ 'master', 'release-*'] @@ -76,10 +76,10 @@ jobs: - id: test-properties uses: ./.github/actions/setup-default-test-properties - precommit-python: + apache-beam: needs: set-properties - name: PreCommit Python - runs-on: [self-hosted, ubuntu-20.04] + name: Run Apache Beam Tests + runs-on: [ self-hosted, ubuntu-20.04 ] strategy: fail-fast: false matrix: @@ -97,179 +97,212 @@ jobs: uses: ./.github/actions/setup-self-hosted-action with: requires-go-18: false - - - name: Run Apache Beam Tests - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/" - - - name: Run Runners Worker Tests - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/worker/" - - - name: Run Testing Tests + - name: run apache_beam/pipeline_test.py uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/testing/" - - - name: Run Transform Tests + arguments: "-Pposargs=apache_beam/pipeline_test.py" + - name: run apache_beam/pvalue_test.py uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/transforms/" + arguments: "-Pposargs=apache_beam/pvalue_test.py" - - name: Run Typehints Tests - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/typehints/" - - name: Run Utils Tests - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/utils/" - - - name: Run Coders Tests - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/coders/" - - - name: Run Internal Tests - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/internal/" - - - name: Run Metrics Tests - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/metrics/" - - - name: Run Options Tests - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/options/" - - - name: Run Runners Portability FnApiRunner Tests - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/portability/fn_api_runner/" - - name: Run Runners Portability Stager Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/portability/stager_test.py" - - name: Run Runners Runner Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/runner_test.py" - - name: Run Runners SDF Utils Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/sdf_utils_test.py" - - - name: Run Tools Microbenchmarks Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/tools/microbenchmarks_test.py" - - - name: Run IO Concat Source Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/io/concat_source_test.py" - - name: Run IO External Generate Sequence Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/io/external/generate_sequence_test.py" - - name: Run IO IOBase Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/io/iobase_test.py" - - name: Run IO Range Trackers Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/io/range_trackers_test.py" - - name: Run IO Restrinction Trackers Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/io/restriction_trackers_test.py" - - name: Run IO Source Test Utils Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/io/source_test_utils_test.py" - - name: Run IO Sources Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/io/sources_test.py" - - name: Run IO Utils Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/io/utils_test.py" - - name: Run IO Watermark Estimators Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/io/watermark_estimators_test.py" - - - name: Run Runners Common Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/common_test.py" - - name: Run Runners Pipeline Context Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/pipeline_context_test.py" - - name: Run Runners Portability Artifact Service Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/portability/artifact_service_test.py" - - - name: Run Runners Portability Job Server Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/portability/job_server_test.py" - - name: Run Runners Local Job Service Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/portability/local_job_service_test.py" - - name: Run Runners Portability Portable Runner Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/portability/portable_runner_test.py" - - name: Run Runners Portability SDK Container Builder Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/runners/portability/sdk_container_builder_test.py" - - - name: Run IO Hadoop File System Test - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/io/hadoopfilesystem_test.py" \ No newline at end of file +# precommit-python: +# needs: set-properties +# name: PreCommit Python +# runs-on: [self-hosted, ubuntu-20.04] +# strategy: +# fail-fast: false +# matrix: +# version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} +# tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} +# steps: +# - name: Checkout code +# uses: actions/checkout@v3 +# with: +# ref: ${{ github.event.pull_request.head.sha }} +# persist-credentials: false +# - name: Set python version +# run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV +# - name: Setup environment +# uses: ./.github/actions/setup-self-hosted-action +# with: +# requires-go-18: false +# +# - name: Run Apache Beam Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/" +# +# - name: Run Runners Worker Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/worker/" +# +# - name: Run Testing Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/testing/" +# +# - name: Run Transform Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/transforms/" +# +# - name: Run Typehints Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/typehints/" +# +# - name: Run Utils Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/utils/" +# +# - name: Run Coders Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/coders/" +# +# - name: Run Internal Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/internal/" +# +# - name: Run Metrics Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/metrics/" +# +# - name: Run Options Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/options/" +# +# - name: Run Runners Portability FnApiRunner Tests +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/portability/fn_api_runner/" +# - name: Run Runners Portability Stager Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/portability/stager_test.py" +# - name: Run Runners Runner Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/runner_test.py" +# - name: Run Runners SDF Utils Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/sdf_utils_test.py" +# +# - name: Run Tools Microbenchmarks Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/tools/microbenchmarks_test.py" +# +# - name: Run IO Concat Source Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/io/concat_source_test.py" +# - name: Run IO External Generate Sequence Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/io/external/generate_sequence_test.py" +# - name: Run IO IOBase Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/io/iobase_test.py" +# - name: Run IO Range Trackers Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/io/range_trackers_test.py" +# - name: Run IO Restrinction Trackers Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/io/restriction_trackers_test.py" +# - name: Run IO Source Test Utils Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/io/source_test_utils_test.py" +# - name: Run IO Sources Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/io/sources_test.py" +# - name: Run IO Utils Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/io/utils_test.py" +# - name: Run IO Watermark Estimators Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/io/watermark_estimators_test.py" +# +# - name: Run Runners Common Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/common_test.py" +# - name: Run Runners Pipeline Context Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/pipeline_context_test.py" +# - name: Run Runners Portability Artifact Service Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/portability/artifact_service_test.py" +# +# - name: Run Runners Portability Job Server Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/portability/job_server_test.py" +# - name: Run Runners Local Job Service Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/portability/local_job_service_test.py" +# - name: Run Runners Portability Portable Runner Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/portability/portable_runner_test.py" +# - name: Run Runners Portability SDK Container Builder Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/runners/portability/sdk_container_builder_test.py" +# +# - name: Run IO Hadoop File System Test +# uses: ./.github/actions/gradle-command-self-hosted-action +# with: +# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} +# arguments: "-Pposargs=apache_beam/io/hadoopfilesystem_test.py" \ No newline at end of file diff --git a/sdks/python/test-suites/tox/common.gradle b/sdks/python/test-suites/tox/common.gradle index 61802ac9c45e..aad145ec1c2d 100644 --- a/sdks/python/test-suites/tox/common.gradle +++ b/sdks/python/test-suites/tox/common.gradle @@ -17,17 +17,18 @@ */ def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') +def posargs = project.hasProperty('posargs') ? project.posargs : '' -toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}" +toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}", posargs test.dependsOn "testPython${pythonVersionSuffix}" -toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud" +toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud", posargs test.dependsOn "testPy${pythonVersionSuffix}Cloud" -toxTask "testPy${pythonVersionSuffix}Dask", "py${pythonVersionSuffix}-dask" +toxTask "testPy${pythonVersionSuffix}Dask", "py${pythonVersionSuffix}-dask", posargs test.dependsOn "testPy${pythonVersionSuffix}Dask" -toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython" +toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython", posargs test.dependsOn "testPy${pythonVersionSuffix}Cython" toxTask "testPy38CloudCoverage", "py38-cloudcoverage" From 5a3c7c52043f35d5a0d67983005177b8a2b188da Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Wed, 30 Nov 2022 16:31:52 -0600 Subject: [PATCH 4/4] Splitted into a workflow per module --- .../action.yml | 4 +- .../setup-self-hosted-action/action.yml | 9 + .../job-precommit-python-apache-beam.yml | 78 +++++ .../workflows/job-precommit-python-coders.yml | 73 +++++ .../job-precommit-python-internal.yml | 73 +++++ .github/workflows/job-precommit-python-io.yml | 128 ++++++++ .../job-precommit-python-metrics.yml | 73 +++++ .../job-precommit-python-options.yml | 73 +++++ .../job-precommit-python-runners.yml | 146 +++++++++ .../job-precommit-python-testing.yml | 73 +++++ .../workflows/job-precommit-python-tools.yml | 73 +++++ .../job-precommit-python-transforms.yml | 73 +++++ .../job-precommit-python-typehints.yml | 73 +++++ .../workflows/job-precommit-python-utils.yml | 73 +++++ .github/workflows/job-precommit-python.yml | 308 ------------------ CI.md | 17 +- 16 files changed, 1035 insertions(+), 312 deletions(-) create mode 100644 .github/workflows/job-precommit-python-apache-beam.yml create mode 100644 .github/workflows/job-precommit-python-coders.yml create mode 100644 .github/workflows/job-precommit-python-internal.yml create mode 100644 .github/workflows/job-precommit-python-io.yml create mode 100644 .github/workflows/job-precommit-python-metrics.yml create mode 100644 .github/workflows/job-precommit-python-options.yml create mode 100644 .github/workflows/job-precommit-python-runners.yml create mode 100644 .github/workflows/job-precommit-python-testing.yml create mode 100644 .github/workflows/job-precommit-python-tools.yml create mode 100644 .github/workflows/job-precommit-python-transforms.yml create mode 100644 .github/workflows/job-precommit-python-typehints.yml create mode 100644 .github/workflows/job-precommit-python-utils.yml delete mode 100644 .github/workflows/job-precommit-python.yml diff --git a/.github/actions/gradle-command-self-hosted-action/action.yml b/.github/actions/gradle-command-self-hosted-action/action.yml index 8fbf63aca42f..cace35dfe0f6 100644 --- a/.github/actions/gradle-command-self-hosted-action/action.yml +++ b/.github/actions/gradle-command-self-hosted-action/action.yml @@ -44,6 +44,8 @@ runs: # Removing settings.xml is a workaround to avoid a decryption issue # of Beam's gradle-command-action plugin and github's provided # maven settings.xml file - rm ~/.m2/settings.xml + if test -f "~/.m2/settings.xml"; then + rm ~/.m2/settings.xml + fi ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} ${{ inputs.arguments }} \ ${{ inputs.default-arguments }} \ No newline at end of file diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml index 27469359e26a..22dfe58567d6 100644 --- a/.github/actions/setup-self-hosted-action/action.yml +++ b/.github/actions/setup-self-hosted-action/action.yml @@ -30,6 +30,10 @@ inputs: required: false description: 'Set as false if does not require py39 setup' default: 'true' + requires-py-310: + required: false + description: 'Set as false if does not require py39 setup' + default: 'true' requires-java-8: required: false description: 'Set as false if does not require java-8 setup' @@ -57,6 +61,11 @@ runs: uses: actions/setup-python@v4 with: python-version: "3.9" + - name: Install python 3.10 + if: ${{ inputs.requires-py-310 == 'true' }} + uses: actions/setup-python@v4 + with: + python-version: "3.10" - name: Set Java Version if: ${{ inputs.requires-java-8 == 'true' }} uses: actions/setup-java@v3 diff --git a/.github/workflows/job-precommit-python-apache-beam.yml b/.github/workflows/job-precommit-python-apache-beam.yml new file mode 100644 index 000000000000..ad4d5934289d --- /dev/null +++ b/.github/workflows/job-precommit-python-apache-beam.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Apache Beam + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + apache-beam: + needs: set-properties + name: Run Apache Beam Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/pipeline_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/pipeline_test.py" + - name: run apache_beam/pvalue_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/pvalue_test.py" diff --git a/.github/workflows/job-precommit-python-coders.yml b/.github/workflows/job-precommit-python-coders.yml new file mode 100644 index 000000000000..ab427fd5644e --- /dev/null +++ b/.github/workflows/job-precommit-python-coders.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Coders + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/coders/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + coders: + needs: set-properties + name: Run Coders Tests + runs-on: [ self-hosted, ubuntu-20.04 ] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/coders + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/coders/" diff --git a/.github/workflows/job-precommit-python-internal.yml b/.github/workflows/job-precommit-python-internal.yml new file mode 100644 index 000000000000..ba54d83ab91d --- /dev/null +++ b/.github/workflows/job-precommit-python-internal.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Internal + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/internal/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + internal: + needs: set-properties + name: Run Internal Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/internal + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/internal/" diff --git a/.github/workflows/job-precommit-python-io.yml b/.github/workflows/job-precommit-python-io.yml new file mode 100644 index 000000000000..9d23392b07e7 --- /dev/null +++ b/.github/workflows/job-precommit-python-io.yml @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python IO + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/io/concat_source_test.py' + - 'sdks/python/apache_beam/io/external/generate_sequence_test.py' + - 'sdks/python/apache_beam/io/hadoopfilesystem_test.py' + - 'sdks/python/apache_beam/io/iobase_test.py' + - 'sdks/python/apache_beam/io/range_trackers_test.py' + - 'sdks/python/apache_beam/io/restriction_trackers_test.py' + - 'sdks/python/apache_beam/io/source_test_utils_test.py' + - 'sdks/python/apache_beam/io/sources_test.py' + - 'sdks/python/apache_beam/io/utils_test.py' + - 'sdks/python/apache_beam/io/watermark_estimators_test.py' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + io: + needs: set-properties + name: Run IO Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/io/contact_source_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/concat_source_test.py" + - name: run apache_beam/io/external/generate_sequence_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/external/generate_sequence_test.py" + - name: run apache_beam/io/hadoopfilesystem_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/hadoopfilesystem_test.py" + - name: run apache_beam/io/iobase_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/iobase_test.py" + - name: run apache_beam/io/range_trackers_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/range_trackers_test.py" + - name: run apache_beam/io/restriction_trackers_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/restriction_trackers_test.py" + - name: run apache_beam/io/source_test_utils_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/source_test_utils_test.py" + - name: run apache_beam/io/sources_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/sources_test.py" + - name: run apache_beam/io/utils_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/utils_test.py" + - name: run apache_beam/io/watermark_estimators_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/watermark_estimators_test.py" + diff --git a/.github/workflows/job-precommit-python-metrics.yml b/.github/workflows/job-precommit-python-metrics.yml new file mode 100644 index 000000000000..8043d59a5ac5 --- /dev/null +++ b/.github/workflows/job-precommit-python-metrics.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Metrics + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/metrics/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + metrics: + needs: set-properties + name: Run Metrics Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/metrics + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/metrics/" diff --git a/.github/workflows/job-precommit-python-options.yml b/.github/workflows/job-precommit-python-options.yml new file mode 100644 index 000000000000..31a4df8fdcce --- /dev/null +++ b/.github/workflows/job-precommit-python-options.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Options + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/options/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + options: + needs: set-properties + name: Run Options Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/options + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/options/" diff --git a/.github/workflows/job-precommit-python-runners.yml b/.github/workflows/job-precommit-python-runners.yml new file mode 100644 index 000000000000..41a90f159222 --- /dev/null +++ b/.github/workflows/job-precommit-python-runners.yml @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Runners + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/runners/common_test.py' + - 'sdks/python/apache_beam/runners/pipeline_context_test.py' + - 'sdks/python/apache_beam/runners/portability/artifact_service_test.py' + - 'sdks/python/apache_beam/runners/portability/fn_api_runner/**' + - 'sdks/python/apache_beam/runners/portability/job_server_test.py' + - 'sdks/python/apache_beam/runners/portability/local_job_service_test.py' + - 'sdks/python/apache_beam/runners/portability/portable_runner_test.py' + - 'sdks/python/apache_beam/runners/portability/sdk_container_builder_test.py' + - 'sdks/python/apache_beam/runners/portability/stager_test.py' + - 'sdks/python/apache_beam/runners/runner_test.py' + - 'sdks/python/apache_beam/runners/sdf_utils_test.py' + - 'sdks/python/apache_beam/runners/worker/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + runners: + needs: set-properties + name: Run Runners Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + + - name: run apache_beam/runners/common_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/common_test.py" + - name: run apache_beam/runners/pipeline_context_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/pipeline_context_test.py" + - name: run apache_beam/runners/portability/artifact_service_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/artifact_service_test.py" + - name: run apache_beam/runners/portability/fn_api_runner/ + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/fn_api_runner/" + - name: Run Runners Portability Job Server Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/job_server_test.py" + - name: Run Runners Local Job Service Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/local_job_service_test.py" + - name: Run Runners Portability Portable Runner Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/portable_runner_test.py" + - name: Run Runners Portability SDK Container Builder Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/sdk_container_builder_test.py" + - name: run apache_beam/runners/portability/stager_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/stager_test.py" + - name: run apache_beam/runners/runner_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/runner_test.py" + - name: run apache_beam/runners/sdf_utils_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/sdf_utils_test.py" + - name: run apache_beam/runners/worker + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/worker/" + + + + + + diff --git a/.github/workflows/job-precommit-python-testing.yml b/.github/workflows/job-precommit-python-testing.yml new file mode 100644 index 000000000000..45f7daf5feb7 --- /dev/null +++ b/.github/workflows/job-precommit-python-testing.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Testing + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/testing/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + testing: + needs: set-properties + name: Run Testing Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/testing + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/testing/" diff --git a/.github/workflows/job-precommit-python-tools.yml b/.github/workflows/job-precommit-python-tools.yml new file mode 100644 index 000000000000..e2e6db353d1d --- /dev/null +++ b/.github/workflows/job-precommit-python-tools.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Tools + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/tools/microbenchmarks_test.py' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + tools: + needs: set-properties + name: Run Tools Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} |ssed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/tools/microbenchmarks_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/tools/microbenchmarks_test.py" diff --git a/.github/workflows/job-precommit-python-transforms.yml b/.github/workflows/job-precommit-python-transforms.yml new file mode 100644 index 000000000000..4cea4f2a7b0e --- /dev/null +++ b/.github/workflows/job-precommit-python-transforms.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Transforms + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: [ 'master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/transforms/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + transforms: + needs: set-properties + name: Run Transforms Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/transforms + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/transforms/" diff --git a/.github/workflows/job-precommit-python-typehints.yml b/.github/workflows/job-precommit-python-typehints.yml new file mode 100644 index 000000000000..34dd5d64e33f --- /dev/null +++ b/.github/workflows/job-precommit-python-typehints.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Typehints + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: [ 'master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/typehints/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + typehints: + needs: set-properties + name: Run Typehints Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/typehints + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/typehints/" diff --git a/.github/workflows/job-precommit-python-utils.yml b/.github/workflows/job-precommit-python-utils.yml new file mode 100644 index 000000000000..b44d7324772e --- /dev/null +++ b/.github/workflows/job-precommit-python-utils.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Utils + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: [ 'master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/utils/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + utils: + needs: set-properties + name: Run Utils Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/utils + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/utils/" diff --git a/.github/workflows/job-precommit-python.yml b/.github/workflows/job-precommit-python.yml deleted file mode 100644 index 8ac68edbf0b1..000000000000 --- a/.github/workflows/job-precommit-python.yml +++ /dev/null @@ -1,308 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: PreCommit Python - -on: - workflow_dispatch: - schedule: - - cron: '0 */6 * * *' - push: - branches: ['master', 'release-*', 'sharding-precommit-python'] - tags: 'v*' - pull_request_target: - branches: [ 'master', 'release-*'] - tags: 'v*' - paths: - - 'sdks/python/apache_beam/**' - - 'sdks/python/apache_beam/coders/**' - - 'sdks/python/apache_beam/internal/**' - - 'sdks/python/apache_beam/io/concat_source_test.py' - - 'sdks/python/apache_beam/io/external/generate_sequence_test.py' - - 'sdks/python/apache_beam/io/hadoopfilesystem_test.py' - - 'sdks/python/apache_beam/io/iobase_test.py' - - 'sdks/python/apache_beam/io/range_trackers_test.py' - - 'sdks/python/apache_beam/io/restriction_trackers_test.py' - - 'sdks/python/apache_beam/io/source_test_utils_test.py' - - 'sdks/python/apache_beam/io/sources_test.py' - - 'sdks/python/apache_beam/io/utils_test.py' - - 'sdks/python/apache_beam/io/watermark_estimators_test.py' - - 'sdks/python/apache_beam/metrics/**' - - 'sdks/python/apache_beam/options/**' - - 'sdks/python/apache_beam/runners/common_test.py' - - 'sdks/python/apache_beam/runners/pipeline_context_test.py' - - 'sdks/python/apache_beam/runners/portability/artifact_service_test.py' - - 'sdks/python/apache_beam/runners/portability/fn_api_runner/**' - - 'sdks/python/apache_beam/runners/portability/job_server_test.py' - - 'sdks/python/apache_beam/runners/portability/local_job_service_test.py' - - 'sdks/python/apache_beam/runners/portability/portable_runner_test.py' - - 'sdks/python/apache_beam/runners/portability/sdk_container_builder_test.py' - - 'sdks/python/apache_beam/runners/portability/stager_test.py' - - 'sdks/python/apache_beam/runners/runner_test.py' - - 'sdks/python/apache_beam/runners/sdf_utils_test.py' - - 'sdks/python/apache_beam/runners/worker/**' - - 'sdks/python/apache_beam/testing/**' - - 'sdks/python/apache_beam/tools/microbenchmarks_test.py' - - 'sdks/python/apache_beam/transforms/**' - - 'sdks/python/apache_beam/typehints/**' - - 'sdks/python/apache_beam/utils/**' -permissions: read-all - -jobs: - set-properties: - runs-on: [self-hosted, ubuntu-20.04] - outputs: - properties: ${{ steps.test-properties.outputs.properties }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - persist-credentials: false - - id: test-properties - uses: ./.github/actions/setup-default-test-properties - - apache-beam: - needs: set-properties - name: Run Apache Beam Tests - runs-on: [ self-hosted, ubuntu-20.04 ] - strategy: - fail-fast: false - matrix: - version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} - tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - persist-credentials: false - - name: Set python version - run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV - - name: Setup environment - uses: ./.github/actions/setup-self-hosted-action - with: - requires-go-18: false - - name: run apache_beam/pipeline_test.py - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/pipeline_test.py" - - name: run apache_beam/pvalue_test.py - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} - arguments: "-Pposargs=apache_beam/pvalue_test.py" - - -# precommit-python: -# needs: set-properties -# name: PreCommit Python -# runs-on: [self-hosted, ubuntu-20.04] -# strategy: -# fail-fast: false -# matrix: -# version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} -# tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} -# steps: -# - name: Checkout code -# uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha }} -# persist-credentials: false -# - name: Set python version -# run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV -# - name: Setup environment -# uses: ./.github/actions/setup-self-hosted-action -# with: -# requires-go-18: false -# -# - name: Run Apache Beam Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/" -# -# - name: Run Runners Worker Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/worker/" -# -# - name: Run Testing Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/testing/" -# -# - name: Run Transform Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/transforms/" -# -# - name: Run Typehints Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/typehints/" -# -# - name: Run Utils Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/utils/" -# -# - name: Run Coders Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/coders/" -# -# - name: Run Internal Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/internal/" -# -# - name: Run Metrics Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/metrics/" -# -# - name: Run Options Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/options/" -# -# - name: Run Runners Portability FnApiRunner Tests -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/portability/fn_api_runner/" -# - name: Run Runners Portability Stager Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/portability/stager_test.py" -# - name: Run Runners Runner Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/runner_test.py" -# - name: Run Runners SDF Utils Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/sdf_utils_test.py" -# -# - name: Run Tools Microbenchmarks Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/tools/microbenchmarks_test.py" -# -# - name: Run IO Concat Source Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/io/concat_source_test.py" -# - name: Run IO External Generate Sequence Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/io/external/generate_sequence_test.py" -# - name: Run IO IOBase Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/io/iobase_test.py" -# - name: Run IO Range Trackers Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/io/range_trackers_test.py" -# - name: Run IO Restrinction Trackers Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/io/restriction_trackers_test.py" -# - name: Run IO Source Test Utils Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/io/source_test_utils_test.py" -# - name: Run IO Sources Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/io/sources_test.py" -# - name: Run IO Utils Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/io/utils_test.py" -# - name: Run IO Watermark Estimators Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/io/watermark_estimators_test.py" -# -# - name: Run Runners Common Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/common_test.py" -# - name: Run Runners Pipeline Context Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/pipeline_context_test.py" -# - name: Run Runners Portability Artifact Service Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/portability/artifact_service_test.py" -# -# - name: Run Runners Portability Job Server Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/portability/job_server_test.py" -# - name: Run Runners Local Job Service Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/portability/local_job_service_test.py" -# - name: Run Runners Portability Portable Runner Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/portability/portable_runner_test.py" -# - name: Run Runners Portability SDK Container Builder Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/runners/portability/sdk_container_builder_test.py" -# -# - name: Run IO Hadoop File System Test -# uses: ./.github/actions/gradle-command-self-hosted-action -# with: -# gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} -# arguments: "-Pposargs=apache_beam/io/hadoopfilesystem_test.py" \ No newline at end of file diff --git a/CI.md b/CI.md index 20e99769467e..e848c38de523 100644 --- a/CI.md +++ b/CI.md @@ -153,9 +153,20 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | PreCommit | Yes | Yes | Yes | Yes | ### PreCommit Workflows -| Workflow | Description | Requires GCP Credentials | -|------------------------------------------------------------------------|---------------------------------------------------|--------------------------| -| [job-precommit-python.yml](.github/workflows/job-precommit-python.yml) | Runs multiple Python test for each of its modules | No | +| Workflow | Description | Requires GCP Credentials | +|------------------------------------------------------------------------------------------------|------------------------------------|--------------------------| +| [job-precommit-python-apache-beam.yml](.github/workflows/job-precommit-python-apache-beam.yml) | Runs Python Apache Beam Unit Tests | No | +| [job-precommit-python-coders.yml](.github/workflows/job-precommit-python-coders.yml) | Runs Python Coders Unit Tests | No | +| [job-precommit-python-internal.yml](.github/workflows/job-precommit-python-internal.yml) | Runs Python Internal Unit Tests | No | +| [job-precommit-python-io.yml](.github/workflows/job-precommit-python-io.yml) | Runs Python IO Unit Tests | No | +| [job-precommit-python-metrics.yml](.github/workflows/job-precommit-python-metrics.yml) | Runs Python Metrics Unit Tests | No | +| [job-precommit-python-options.yml](.github/workflows/job-precommit-python-options.yml) | Runs Python Options Unit Tests | No | +| [job-precommit-python-runners.yml](.github/workflows/job-precommit-python-runners.yml) | Runs Python Runners Unit Tests | No | +| [job-precommit-python-testing.yml](.github/workflows/job-precommit-python-testing.yml) | Runs Python Testing Unit Tests | No | +| [job-precommit-python-tools.yml](.github/workflows/job-precommit-python-tools.yml) | Runs Python Tools Unit Tests | No | +| [job-precommit-python-transforms.yml](.github/workflows/job-precommit-python-transforms.yml) | Runs Python Transforms Unit Tests | No | +| [job-precommit-python-typehints.yml](.github/workflows/job-precommit-python-typehints.yml) | Runs Python Typehints Unit Tests | No | +| [job-precommit-python-utils.yml](.github/workflows/job-precommit-python-utils.yml) | Runs Python Utils Unit Tests | No | ### PostCommit Workflows