diff --git a/.github/actions/gradle-command-self-hosted-action/action.yml b/.github/actions/gradle-command-self-hosted-action/action.yml index 8fbf63aca42f..cace35dfe0f6 100644 --- a/.github/actions/gradle-command-self-hosted-action/action.yml +++ b/.github/actions/gradle-command-self-hosted-action/action.yml @@ -44,6 +44,8 @@ runs: # Removing settings.xml is a workaround to avoid a decryption issue # of Beam's gradle-command-action plugin and github's provided # maven settings.xml file - rm ~/.m2/settings.xml + if test -f "~/.m2/settings.xml"; then + rm ~/.m2/settings.xml + fi ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} ${{ inputs.arguments }} \ ${{ inputs.default-arguments }} \ No newline at end of file diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml index 27469359e26a..22dfe58567d6 100644 --- a/.github/actions/setup-self-hosted-action/action.yml +++ b/.github/actions/setup-self-hosted-action/action.yml @@ -30,6 +30,10 @@ inputs: required: false description: 'Set as false if does not require py39 setup' default: 'true' + requires-py-310: + required: false + description: 'Set as false if does not require py39 setup' + default: 'true' requires-java-8: required: false description: 'Set as false if does not require java-8 setup' @@ -57,6 +61,11 @@ runs: uses: actions/setup-python@v4 with: python-version: "3.9" + - name: Install python 3.10 + if: ${{ inputs.requires-py-310 == 'true' }} + uses: actions/setup-python@v4 + with: + python-version: "3.10" - name: Set Java Version if: ${{ inputs.requires-java-8 == 'true' }} uses: actions/setup-java@v3 diff --git a/.github/workflows/job-precommit-python-apache-beam.yml b/.github/workflows/job-precommit-python-apache-beam.yml new file mode 100644 index 000000000000..ad4d5934289d --- /dev/null +++ b/.github/workflows/job-precommit-python-apache-beam.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Apache Beam + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + apache-beam: + needs: set-properties + name: Run Apache Beam Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/pipeline_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/pipeline_test.py" + - name: run apache_beam/pvalue_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/pvalue_test.py" diff --git a/.github/workflows/job-precommit-python-coders.yml b/.github/workflows/job-precommit-python-coders.yml new file mode 100644 index 000000000000..ab427fd5644e --- /dev/null +++ b/.github/workflows/job-precommit-python-coders.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Coders + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/coders/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + coders: + needs: set-properties + name: Run Coders Tests + runs-on: [ self-hosted, ubuntu-20.04 ] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/coders + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/coders/" diff --git a/.github/workflows/job-precommit-python-internal.yml b/.github/workflows/job-precommit-python-internal.yml new file mode 100644 index 000000000000..ba54d83ab91d --- /dev/null +++ b/.github/workflows/job-precommit-python-internal.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Internal + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/internal/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + internal: + needs: set-properties + name: Run Internal Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/internal + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/internal/" diff --git a/.github/workflows/job-precommit-python-io.yml b/.github/workflows/job-precommit-python-io.yml new file mode 100644 index 000000000000..9d23392b07e7 --- /dev/null +++ b/.github/workflows/job-precommit-python-io.yml @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python IO + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/io/concat_source_test.py' + - 'sdks/python/apache_beam/io/external/generate_sequence_test.py' + - 'sdks/python/apache_beam/io/hadoopfilesystem_test.py' + - 'sdks/python/apache_beam/io/iobase_test.py' + - 'sdks/python/apache_beam/io/range_trackers_test.py' + - 'sdks/python/apache_beam/io/restriction_trackers_test.py' + - 'sdks/python/apache_beam/io/source_test_utils_test.py' + - 'sdks/python/apache_beam/io/sources_test.py' + - 'sdks/python/apache_beam/io/utils_test.py' + - 'sdks/python/apache_beam/io/watermark_estimators_test.py' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + io: + needs: set-properties + name: Run IO Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/io/contact_source_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/concat_source_test.py" + - name: run apache_beam/io/external/generate_sequence_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/external/generate_sequence_test.py" + - name: run apache_beam/io/hadoopfilesystem_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/hadoopfilesystem_test.py" + - name: run apache_beam/io/iobase_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/iobase_test.py" + - name: run apache_beam/io/range_trackers_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/range_trackers_test.py" + - name: run apache_beam/io/restriction_trackers_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/restriction_trackers_test.py" + - name: run apache_beam/io/source_test_utils_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/source_test_utils_test.py" + - name: run apache_beam/io/sources_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/sources_test.py" + - name: run apache_beam/io/utils_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/utils_test.py" + - name: run apache_beam/io/watermark_estimators_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/watermark_estimators_test.py" + diff --git a/.github/workflows/job-precommit-python-metrics.yml b/.github/workflows/job-precommit-python-metrics.yml new file mode 100644 index 000000000000..8043d59a5ac5 --- /dev/null +++ b/.github/workflows/job-precommit-python-metrics.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Metrics + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/metrics/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + metrics: + needs: set-properties + name: Run Metrics Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/metrics + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/metrics/" diff --git a/.github/workflows/job-precommit-python-options.yml b/.github/workflows/job-precommit-python-options.yml new file mode 100644 index 000000000000..31a4df8fdcce --- /dev/null +++ b/.github/workflows/job-precommit-python-options.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Options + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/options/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + options: + needs: set-properties + name: Run Options Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/options + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/options/" diff --git a/.github/workflows/job-precommit-python-runners.yml b/.github/workflows/job-precommit-python-runners.yml new file mode 100644 index 000000000000..41a90f159222 --- /dev/null +++ b/.github/workflows/job-precommit-python-runners.yml @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Runners + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/runners/common_test.py' + - 'sdks/python/apache_beam/runners/pipeline_context_test.py' + - 'sdks/python/apache_beam/runners/portability/artifact_service_test.py' + - 'sdks/python/apache_beam/runners/portability/fn_api_runner/**' + - 'sdks/python/apache_beam/runners/portability/job_server_test.py' + - 'sdks/python/apache_beam/runners/portability/local_job_service_test.py' + - 'sdks/python/apache_beam/runners/portability/portable_runner_test.py' + - 'sdks/python/apache_beam/runners/portability/sdk_container_builder_test.py' + - 'sdks/python/apache_beam/runners/portability/stager_test.py' + - 'sdks/python/apache_beam/runners/runner_test.py' + - 'sdks/python/apache_beam/runners/sdf_utils_test.py' + - 'sdks/python/apache_beam/runners/worker/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + runners: + needs: set-properties + name: Run Runners Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + + - name: run apache_beam/runners/common_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/common_test.py" + - name: run apache_beam/runners/pipeline_context_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/pipeline_context_test.py" + - name: run apache_beam/runners/portability/artifact_service_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/artifact_service_test.py" + - name: run apache_beam/runners/portability/fn_api_runner/ + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/fn_api_runner/" + - name: Run Runners Portability Job Server Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/job_server_test.py" + - name: Run Runners Local Job Service Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/local_job_service_test.py" + - name: Run Runners Portability Portable Runner Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/portable_runner_test.py" + - name: Run Runners Portability SDK Container Builder Test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/sdk_container_builder_test.py" + - name: run apache_beam/runners/portability/stager_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/portability/stager_test.py" + - name: run apache_beam/runners/runner_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/runner_test.py" + - name: run apache_beam/runners/sdf_utils_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/sdf_utils_test.py" + - name: run apache_beam/runners/worker + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/runners/worker/" + + + + + + diff --git a/.github/workflows/job-precommit-python-testing.yml b/.github/workflows/job-precommit-python-testing.yml new file mode 100644 index 000000000000..45f7daf5feb7 --- /dev/null +++ b/.github/workflows/job-precommit-python-testing.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Testing + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/testing/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + testing: + needs: set-properties + name: Run Testing Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/testing + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/testing/" diff --git a/.github/workflows/job-precommit-python-tools.yml b/.github/workflows/job-precommit-python-tools.yml new file mode 100644 index 000000000000..e2e6db353d1d --- /dev/null +++ b/.github/workflows/job-precommit-python-tools.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Tools + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/tools/microbenchmarks_test.py' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + tools: + needs: set-properties + name: Run Tools Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} |ssed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/tools/microbenchmarks_test.py + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/tools/microbenchmarks_test.py" diff --git a/.github/workflows/job-precommit-python-transforms.yml b/.github/workflows/job-precommit-python-transforms.yml new file mode 100644 index 000000000000..4cea4f2a7b0e --- /dev/null +++ b/.github/workflows/job-precommit-python-transforms.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Transforms + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: [ 'master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/transforms/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + transforms: + needs: set-properties + name: Run Transforms Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/transforms + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/transforms/" diff --git a/.github/workflows/job-precommit-python-typehints.yml b/.github/workflows/job-precommit-python-typehints.yml new file mode 100644 index 000000000000..34dd5d64e33f --- /dev/null +++ b/.github/workflows/job-precommit-python-typehints.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Typehints + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: [ 'master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/typehints/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + typehints: + needs: set-properties + name: Run Typehints Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/typehints + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/typehints/" diff --git a/.github/workflows/job-precommit-python-utils.yml b/.github/workflows/job-precommit-python-utils.yml new file mode 100644 index 000000000000..b44d7324772e --- /dev/null +++ b/.github/workflows/job-precommit-python-utils.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PreCommit Python Utils + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: [ 'master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*' ] + tags: 'v*' + paths: + - 'sdks/python/apache_beam/utils/**' +permissions: read-all + +jobs: + set-properties: + runs-on: [self-hosted, ubuntu-20.04] + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + utils: + needs: set-properties + name: Run Utils Tests + runs-on: [self-hosted, ubuntu-20.04] + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: run apache_beam/utils + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/utils/" diff --git a/CI.md b/CI.md index cb26171654b2..e848c38de523 100644 --- a/CI.md +++ b/CI.md @@ -153,10 +153,21 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | PreCommit | Yes | Yes | Yes | Yes | ### PreCommit Workflows +| Workflow | Description | Requires GCP Credentials | +|------------------------------------------------------------------------------------------------|------------------------------------|--------------------------| +| [job-precommit-python-apache-beam.yml](.github/workflows/job-precommit-python-apache-beam.yml) | Runs Python Apache Beam Unit Tests | No | +| [job-precommit-python-coders.yml](.github/workflows/job-precommit-python-coders.yml) | Runs Python Coders Unit Tests | No | +| [job-precommit-python-internal.yml](.github/workflows/job-precommit-python-internal.yml) | Runs Python Internal Unit Tests | No | +| [job-precommit-python-io.yml](.github/workflows/job-precommit-python-io.yml) | Runs Python IO Unit Tests | No | +| [job-precommit-python-metrics.yml](.github/workflows/job-precommit-python-metrics.yml) | Runs Python Metrics Unit Tests | No | +| [job-precommit-python-options.yml](.github/workflows/job-precommit-python-options.yml) | Runs Python Options Unit Tests | No | +| [job-precommit-python-runners.yml](.github/workflows/job-precommit-python-runners.yml) | Runs Python Runners Unit Tests | No | +| [job-precommit-python-testing.yml](.github/workflows/job-precommit-python-testing.yml) | Runs Python Testing Unit Tests | No | +| [job-precommit-python-tools.yml](.github/workflows/job-precommit-python-tools.yml) | Runs Python Tools Unit Tests | No | +| [job-precommit-python-transforms.yml](.github/workflows/job-precommit-python-transforms.yml) | Runs Python Transforms Unit Tests | No | +| [job-precommit-python-typehints.yml](.github/workflows/job-precommit-python-typehints.yml) | Runs Python Typehints Unit Tests | No | +| [job-precommit-python-utils.yml](.github/workflows/job-precommit-python-utils.yml) | Runs Python Utils Unit Tests | No | -| Workflow | Description | Requires GCP Credentials | -|----------------------------------------------------------------------------------|-------------------------|---------------------------| -| [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | ### PostCommit Workflows diff --git a/sdks/python/test-suites/tox/common.gradle b/sdks/python/test-suites/tox/common.gradle index 61802ac9c45e..aad145ec1c2d 100644 --- a/sdks/python/test-suites/tox/common.gradle +++ b/sdks/python/test-suites/tox/common.gradle @@ -17,17 +17,18 @@ */ def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') +def posargs = project.hasProperty('posargs') ? project.posargs : '' -toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}" +toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}", posargs test.dependsOn "testPython${pythonVersionSuffix}" -toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud" +toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud", posargs test.dependsOn "testPy${pythonVersionSuffix}Cloud" -toxTask "testPy${pythonVersionSuffix}Dask", "py${pythonVersionSuffix}-dask" +toxTask "testPy${pythonVersionSuffix}Dask", "py${pythonVersionSuffix}-dask", posargs test.dependsOn "testPy${pythonVersionSuffix}Dask" -toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython" +toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython", posargs test.dependsOn "testPy${pythonVersionSuffix}Cython" toxTask "testPy38CloudCoverage", "py38-cloudcoverage"