diff --git a/.github/workflows/job-precommit-python-avro-io.yml b/.github/workflows/job-precommit-python-avro-io.yml new file mode 100644 index 000000000000..c27d8a205736 --- /dev/null +++ b/.github/workflows/job-precommit-python-avro-io.yml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python Avro IO class. + +name: Python Avro IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request: + branches: ['master', 'release-*'] + tags: ['v*'] +# paths: ['sdks/python/apache_beam/io/avroio.py', +# 'sdks/python/apache_beam/io/avroio_test.py'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_avro_io: + needs: set-properties + name: Python Avro IO + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/avroio_test.py" diff --git a/.github/workflows/job-precommit-python-azure-io.yml b/.github/workflows/job-precommit-python-azure-io.yml new file mode 100644 index 000000000000..7e32b13cce7e --- /dev/null +++ b/.github/workflows/job-precommit-python-azure-io.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python Azure IO class. + +name: Python Azure IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request: + branches: ['master', 'release-*'] + tags: ['v*'] +# paths: ['sdks/python/apache_beam/io/azure/**'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_azure_io: + needs: set-properties + name: Python Azure IO + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/azure/" \ No newline at end of file diff --git a/.github/workflows/job-precommit-python-flink-io.yml b/.github/workflows/job-precommit-python-flink-io.yml new file mode 100644 index 000000000000..4e0797b13a4c --- /dev/null +++ b/.github/workflows/job-precommit-python-flink-io.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python Flink IO class. + +name: Python Flink IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request: + branches: ['master', 'release-*'] + tags: ['v*'] +# paths: ['sdks/python/apache_beam/io/flink/**'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_flink_io: + needs: set-properties + name: Python Flink IO + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/flink/" \ No newline at end of file diff --git a/.github/workflows/job-precommit-python-gcp-io.yml b/.github/workflows/job-precommit-python-gcp-io.yml new file mode 100644 index 000000000000..005507e56b29 --- /dev/null +++ b/.github/workflows/job-precommit-python-gcp-io.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python GCP IO classes. + +name: Python GCP IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request: + branches: ['master', 'release-*'] + tags: ['v*'] +# paths: ['sdks/python/apache_beam/io/gcp/**'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_gcp_io: + needs: set-properties + name: Python GCP IO + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/gcp/" \ No newline at end of file diff --git a/.github/workflows/job-precommit-python-hadoop-io.yml b/.github/workflows/job-precommit-python-hadoop-io.yml new file mode 100644 index 000000000000..5f403e17f971 --- /dev/null +++ b/.github/workflows/job-precommit-python-hadoop-io.yml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python Hadoop IO class. + +name: Python Hadoop IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request: + branches: ['master', 'release-*'] + tags: ['v*'] +# paths: ['sdks/python/apache_beam/io/hadoopfilesystem.py', +# 'sdks/python/apache_beam/io/hadoopfilesystem_test.py'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_hadoop_io: + needs: set-properties + name: Python Hadoop IO + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/hadoopfilesystem_test.py" \ No newline at end of file diff --git a/.github/workflows/job-precommit-python-ml.yml b/.github/workflows/job-precommit-python-ml.yml new file mode 100644 index 000000000000..d0d2b4a9d8c0 --- /dev/null +++ b/.github/workflows/job-precommit-python-ml.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python ML classes. + +name: Python ML + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request: + branches: ['master', 'release-*'] + tags: ['v*'] +# paths: ['sdks/python/apache_beam/ml/**'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_ml: + needs: set-properties + name: Python ML + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/ml/" \ No newline at end of file diff --git a/CI.md b/CI.md index ee911584f7f0..4671364ffd7e 100644 --- a/CI.md +++ b/CI.md @@ -133,9 +133,14 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | PreCommit | Yes | Yes | Yes | Yes | ### PreCommit Workflows -| Workflow | Description | Requires GCP Credentials | -|----------------------------------------------------------------------------------|-------------------------|---------------------------| -| [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | +| Workflow | Description | Requires GCP Credentials | +|--------------------------------------------------------------------------------------------|----------------------------------|--------------------------| +| [job-precommit-python-avro-io.yml](.github/workflows/job-precommit-python-avro-io.yml) | Runs Python Avro IO Unit Tests | No | +| [job-precommit-python-azure-io.yml](.github/workflows/job-precommit-python-azure-io.yml) | Runs Python Azure IO Unit Tests | No | +| [job-precommit-python-flink-io.yml](.github/workflows/job-precommit-python-flink-io.yml) | Runs Python Flink IO Unit Tests | No | +| [job-precommit-python-gcp-io.yml](.github/workflows/job-precommit-python-gcp-io.yml) | Runs Python GCP IO Unit Tests | No | +| [job-precommit-python-hadoop-io.yml](.github/workflows/job-precommit-python-hadoop-io.yml) | Runs Python Hadoop IO Unit Tests | No | +| [job-precommit-python-ml.yml](.github/workflows/job-precommit-python-ml.yml) | Runs Python ML Unit Tests | No | ### PostCommit Workflows | Workflow | Description | Requires GCP Credentials | diff --git a/sdks/python/test-suites/tox/common.gradle b/sdks/python/test-suites/tox/common.gradle index 99afc1d72557..c9b707575aba 100644 --- a/sdks/python/test-suites/tox/common.gradle +++ b/sdks/python/test-suites/tox/common.gradle @@ -17,14 +17,15 @@ */ def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') +def posargs = project.hasProperty('posargs') ? project.posargs : '' -toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}" +toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}", posargs test.dependsOn "testPython${pythonVersionSuffix}" -toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud" +toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud", posargs test.dependsOn "testPy${pythonVersionSuffix}Cloud" -toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython" +toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython", posargs test.dependsOn "testPy${pythonVersionSuffix}Cython" toxTask "testPy38CloudCoverage", "py38-cloudcoverage"