From f71107b8fc7191e3bdbfa6d2023594ab899a15a0 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Wed, 14 Sep 2022 16:45:20 -0500 Subject: [PATCH 1/6] Migrate Python Precommit IOs jobs from Jenkins to GA --- .../job-precommit-python-avro-io.yml | 74 ++++++++++++++++++ .../job-precommit-python-azure-io.yml | 75 +++++++++++++++++++ .../job-precommit-python-flink-io.yml | 74 ++++++++++++++++++ .../workflows/job-precommit-python-gcp-io.yml | 74 ++++++++++++++++++ .../job-precommit-python-hadoop-io.yml | 75 +++++++++++++++++++ .github/workflows/job-precommit-python-ml.yml | 74 ++++++++++++++++++ 6 files changed, 446 insertions(+) create mode 100644 .github/workflows/job-precommit-python-avro-io.yml create mode 100644 .github/workflows/job-precommit-python-azure-io.yml create mode 100644 .github/workflows/job-precommit-python-flink-io.yml create mode 100644 .github/workflows/job-precommit-python-gcp-io.yml create mode 100644 .github/workflows/job-precommit-python-hadoop-io.yml create mode 100644 .github/workflows/job-precommit-python-ml.yml diff --git a/.github/workflows/job-precommit-python-avro-io.yml b/.github/workflows/job-precommit-python-avro-io.yml new file mode 100644 index 000000000000..9f232822df8e --- /dev/null +++ b/.github/workflows/job-precommit-python-avro-io.yml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python Avro IO class. + +name: Python Avro IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request_target: + branches: ['master', 'release-*'] + tags: ['v*'] + paths: ['sdks/python/apache_beam/io/avroio.py', + 'sdks/python/apache_beam/io/avroio_test.py'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_avro_io: + needs: set-properties + name: Python Avro IO + runs-on: self-hosted + strategy: + fail-fast: true + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/avroio_test.py" diff --git a/.github/workflows/job-precommit-python-azure-io.yml b/.github/workflows/job-precommit-python-azure-io.yml new file mode 100644 index 000000000000..04aa62628df0 --- /dev/null +++ b/.github/workflows/job-precommit-python-azure-io.yml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python Azure IO class. + +name: Python Azure IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request_target: + branches: ['master', 'release-*'] + tags: ['v*'] + paths: ['sdks/python/apache_beam/io/azure/**'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_azure_io: + needs: set-properties + name: Python Azure IO + runs-on: self-hosted + strategy: + fail-fast: true + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/azure/" \ No newline at end of file diff --git a/.github/workflows/job-precommit-python-flink-io.yml b/.github/workflows/job-precommit-python-flink-io.yml new file mode 100644 index 000000000000..1dd376560339 --- /dev/null +++ b/.github/workflows/job-precommit-python-flink-io.yml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python Flink IO class. + +name: Python Flink IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request_target: + branches: ['master', 'release-*'] + tags: ['v*'] + paths: ['sdks/python/apache_beam/io/flink/**'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_flink_io: + needs: set-properties + name: Python Flink IO + runs-on: self-hosted + strategy: + fail-fast: true + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/flink/" \ No newline at end of file diff --git a/.github/workflows/job-precommit-python-gcp-io.yml b/.github/workflows/job-precommit-python-gcp-io.yml new file mode 100644 index 000000000000..1e25423c2337 --- /dev/null +++ b/.github/workflows/job-precommit-python-gcp-io.yml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python GCP IO classes. + +name: Python GCP IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request_target: + branches: ['master', 'release-*'] + tags: ['v*'] + paths: ['sdks/python/apache_beam/io/gcp/**'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_gcp_io: + needs: set-properties + name: Python GCP IO + runs-on: self-hosted + strategy: + fail-fast: true + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/gcp/" \ No newline at end of file diff --git a/.github/workflows/job-precommit-python-hadoop-io.yml b/.github/workflows/job-precommit-python-hadoop-io.yml new file mode 100644 index 000000000000..0e3b048ecdef --- /dev/null +++ b/.github/workflows/job-precommit-python-hadoop-io.yml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python Hadoop IO class. + +name: Python Hadoop IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request_target: + branches: ['master', 'release-*'] + tags: ['v*'] + paths: ['sdks/python/apache_beam/io/hadoopfilesystem.py', + 'sdks/python/apache_beam/io/hadoopfilesystem_test.py'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_hadoop_io: + needs: set-properties + name: Python Hadoop IO + runs-on: self-hosted + strategy: + fail-fast: true + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/io/hadoopfilesystem_test.py" \ No newline at end of file diff --git a/.github/workflows/job-precommit-python-ml.yml b/.github/workflows/job-precommit-python-ml.yml new file mode 100644 index 000000000000..674c3ddb61da --- /dev/null +++ b/.github/workflows/job-precommit-python-ml.yml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the test of python ML classes. + +name: Python ML + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request_target: + branches: ['master', 'release-*'] + tags: ['v*'] + paths: ['sdks/python/apache_beam/ml/**'] +permissions: read-all + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + python_ml: + needs: set-properties + name: Python ML + runs-on: self-hosted + strategy: + fail-fast: true + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run Tox Tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{env.PYTHON_VERSION}}:testPy${{env.PYTHON_VERSION}}${{matrix.tox-env}} + arguments: "-Pposargs=apache_beam/ml/" \ No newline at end of file From c5d7c836a930c5d8caf4d9eace29c690a909b5f9 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Wed, 14 Sep 2022 17:04:17 -0500 Subject: [PATCH 2/6] Add documentation for new jobs to CI.md --- CI.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/CI.md b/CI.md index ed75a4dd4433..6ab5d47ec7ef 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,38 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PreCommit Workflows + +#### Python Avro IO - [job-precommit-python-avro-io.yml](.github/workflows/job-precommit-python-avro-io.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|----------------|--------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Avro IO | Runs Python Avro IO Unit Tests | Yes | Yes | Yes | - | + +#### Python Azure IO - [job-precommit-python-azure-io.yml](.github/workflows/job-precommit-python-azure-io.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-----------------|---------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Azure IO | Runs Python Azure IO Unit Tests | Yes | Yes | Yes | - | + +#### Python Flink IO - [job-precommit-python-flink-io.yml](.github/workflows/job-precommit-python-flink-io.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-----------------|---------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Flink IO | Runs Python Flink IO Unit Tests | Yes | Yes | Yes | - | + +#### Python GCP IO - [job-precommit-python-gcp-io.yml](.github/workflows/job-precommit-python-gcp-io.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|---------------|-------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python GCP IO | Runs Python GCP IO Unit Tests | Yes | Yes | Yes | - | + +#### Python Hadoop IO - [job-precommit-python-hadoop-io.yml](.github/workflows/job-precommit-python-hadoop-io.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------|----------------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Hadoop IO | Runs Python Hadoop IO Unit Tests | Yes | Yes | Yes | - | + +#### Python ML - [job-precommit-python-ml.yml](.github/workflows/job-precommit-python-ml.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-----------|---------------------------|------------------|-----------------------|---------------|--------------------------| +| Python ML | Runs Python ML Unit Tests | Yes | Yes | Yes | - | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From 3cb185f36e1843bfad7afc00c15c4a1985d2a794 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Wed, 14 Sep 2022 17:13:07 -0500 Subject: [PATCH 3/6] Add args to tox test-suite to run different tests by directory --- sdks/python/test-suites/tox/common.gradle | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sdks/python/test-suites/tox/common.gradle b/sdks/python/test-suites/tox/common.gradle index 99afc1d72557..c9b707575aba 100644 --- a/sdks/python/test-suites/tox/common.gradle +++ b/sdks/python/test-suites/tox/common.gradle @@ -17,14 +17,15 @@ */ def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') +def posargs = project.hasProperty('posargs') ? project.posargs : '' -toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}" +toxTask "testPython${pythonVersionSuffix}", "py${pythonVersionSuffix}", posargs test.dependsOn "testPython${pythonVersionSuffix}" -toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud" +toxTask "testPy${pythonVersionSuffix}Cloud", "py${pythonVersionSuffix}-cloud", posargs test.dependsOn "testPy${pythonVersionSuffix}Cloud" -toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython" +toxTask "testPy${pythonVersionSuffix}Cython", "py${pythonVersionSuffix}-cython", posargs test.dependsOn "testPy${pythonVersionSuffix}Cython" toxTask "testPy38CloudCoverage", "py38-cloudcoverage" From bc55a1ed4739f997c6c8a2dbeead9d138a0018e8 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Tue, 20 Sep 2022 11:30:43 -0500 Subject: [PATCH 4/6] Testing --- .github/workflows/job-precommit-python-avro-io.yml | 6 +++--- .github/workflows/job-precommit-python-azure-io.yml | 4 ++-- .github/workflows/job-precommit-python-flink-io.yml | 4 ++-- .github/workflows/job-precommit-python-gcp-io.yml | 4 ++-- .github/workflows/job-precommit-python-hadoop-io.yml | 6 +++--- .github/workflows/job-precommit-python-ml.yml | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/job-precommit-python-avro-io.yml b/.github/workflows/job-precommit-python-avro-io.yml index 9f232822df8e..3b6f673db383 100644 --- a/.github/workflows/job-precommit-python-avro-io.yml +++ b/.github/workflows/job-precommit-python-avro-io.yml @@ -25,11 +25,11 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] - pull_request_target: + pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/python/apache_beam/io/avroio.py', - 'sdks/python/apache_beam/io/avroio_test.py'] +# paths: ['sdks/python/apache_beam/io/avroio.py', +# 'sdks/python/apache_beam/io/avroio_test.py'] permissions: read-all jobs: diff --git a/.github/workflows/job-precommit-python-azure-io.yml b/.github/workflows/job-precommit-python-azure-io.yml index 04aa62628df0..f5567d550d93 100644 --- a/.github/workflows/job-precommit-python-azure-io.yml +++ b/.github/workflows/job-precommit-python-azure-io.yml @@ -25,10 +25,10 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] - pull_request_target: + pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/python/apache_beam/io/azure/**'] +# paths: ['sdks/python/apache_beam/io/azure/**'] permissions: read-all jobs: diff --git a/.github/workflows/job-precommit-python-flink-io.yml b/.github/workflows/job-precommit-python-flink-io.yml index 1dd376560339..1393abc86d9a 100644 --- a/.github/workflows/job-precommit-python-flink-io.yml +++ b/.github/workflows/job-precommit-python-flink-io.yml @@ -25,10 +25,10 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] - pull_request_target: + pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/python/apache_beam/io/flink/**'] +# paths: ['sdks/python/apache_beam/io/flink/**'] permissions: read-all jobs: diff --git a/.github/workflows/job-precommit-python-gcp-io.yml b/.github/workflows/job-precommit-python-gcp-io.yml index 1e25423c2337..487c81dcd028 100644 --- a/.github/workflows/job-precommit-python-gcp-io.yml +++ b/.github/workflows/job-precommit-python-gcp-io.yml @@ -25,10 +25,10 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] - pull_request_target: + pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/python/apache_beam/io/gcp/**'] +# paths: ['sdks/python/apache_beam/io/gcp/**'] permissions: read-all jobs: diff --git a/.github/workflows/job-precommit-python-hadoop-io.yml b/.github/workflows/job-precommit-python-hadoop-io.yml index 0e3b048ecdef..230fefc77ebb 100644 --- a/.github/workflows/job-precommit-python-hadoop-io.yml +++ b/.github/workflows/job-precommit-python-hadoop-io.yml @@ -25,11 +25,11 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] - pull_request_target: + pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/python/apache_beam/io/hadoopfilesystem.py', - 'sdks/python/apache_beam/io/hadoopfilesystem_test.py'] +# paths: ['sdks/python/apache_beam/io/hadoopfilesystem.py', +# 'sdks/python/apache_beam/io/hadoopfilesystem_test.py'] permissions: read-all jobs: diff --git a/.github/workflows/job-precommit-python-ml.yml b/.github/workflows/job-precommit-python-ml.yml index 674c3ddb61da..a1c8d53f984d 100644 --- a/.github/workflows/job-precommit-python-ml.yml +++ b/.github/workflows/job-precommit-python-ml.yml @@ -25,10 +25,10 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] - pull_request_target: + pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/python/apache_beam/ml/**'] +# paths: ['sdks/python/apache_beam/ml/**'] permissions: read-all jobs: From c5ecc4a4c67736d0465641a12f068eee381c34ed Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Tue, 20 Sep 2022 13:31:35 -0500 Subject: [PATCH 5/6] Add tox-env to matrix --- .github/workflows/job-precommit-python-flink-io.yml | 1 + .github/workflows/job-precommit-python-gcp-io.yml | 1 + .github/workflows/job-precommit-python-hadoop-io.yml | 1 + .github/workflows/job-precommit-python-ml.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/job-precommit-python-flink-io.yml b/.github/workflows/job-precommit-python-flink-io.yml index 1393abc86d9a..dcb992233245 100644 --- a/.github/workflows/job-precommit-python-flink-io.yml +++ b/.github/workflows/job-precommit-python-flink-io.yml @@ -54,6 +54,7 @@ jobs: fail-fast: true matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/.github/workflows/job-precommit-python-gcp-io.yml b/.github/workflows/job-precommit-python-gcp-io.yml index 487c81dcd028..85b4640b2b92 100644 --- a/.github/workflows/job-precommit-python-gcp-io.yml +++ b/.github/workflows/job-precommit-python-gcp-io.yml @@ -54,6 +54,7 @@ jobs: fail-fast: true matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/.github/workflows/job-precommit-python-hadoop-io.yml b/.github/workflows/job-precommit-python-hadoop-io.yml index 230fefc77ebb..82198d0b9073 100644 --- a/.github/workflows/job-precommit-python-hadoop-io.yml +++ b/.github/workflows/job-precommit-python-hadoop-io.yml @@ -55,6 +55,7 @@ jobs: fail-fast: true matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/.github/workflows/job-precommit-python-ml.yml b/.github/workflows/job-precommit-python-ml.yml index a1c8d53f984d..ccf15768c9b2 100644 --- a/.github/workflows/job-precommit-python-ml.yml +++ b/.github/workflows/job-precommit-python-ml.yml @@ -54,6 +54,7 @@ jobs: fail-fast: true matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} steps: - name: Checkout code uses: actions/checkout@v3 From 2f738018f3ed9e8e83c378116636196aa2ee9f1c Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 29 Sep 2022 13:15:52 -0500 Subject: [PATCH 6/6] Testing fail-fast=false and remove submodules: recursive --- .github/workflows/job-precommit-python-avro-io.yml | 4 +--- .github/workflows/job-precommit-python-azure-io.yml | 4 +--- .github/workflows/job-precommit-python-flink-io.yml | 4 +--- .github/workflows/job-precommit-python-gcp-io.yml | 4 +--- .github/workflows/job-precommit-python-hadoop-io.yml | 4 +--- .github/workflows/job-precommit-python-ml.yml | 4 +--- 6 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/workflows/job-precommit-python-avro-io.yml b/.github/workflows/job-precommit-python-avro-io.yml index 3b6f673db383..c27d8a205736 100644 --- a/.github/workflows/job-precommit-python-avro-io.yml +++ b/.github/workflows/job-precommit-python-avro-io.yml @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -52,7 +51,7 @@ jobs: name: Python Avro IO runs-on: self-hosted strategy: - fail-fast: true + fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} @@ -62,7 +61,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Set python version run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV - name: Setup environment diff --git a/.github/workflows/job-precommit-python-azure-io.yml b/.github/workflows/job-precommit-python-azure-io.yml index f5567d550d93..7e32b13cce7e 100644 --- a/.github/workflows/job-precommit-python-azure-io.yml +++ b/.github/workflows/job-precommit-python-azure-io.yml @@ -42,7 +42,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -51,7 +50,7 @@ jobs: name: Python Azure IO runs-on: self-hosted strategy: - fail-fast: true + fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} @@ -61,7 +60,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Set python version run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV - name: Setup environment diff --git a/.github/workflows/job-precommit-python-flink-io.yml b/.github/workflows/job-precommit-python-flink-io.yml index dcb992233245..4e0797b13a4c 100644 --- a/.github/workflows/job-precommit-python-flink-io.yml +++ b/.github/workflows/job-precommit-python-flink-io.yml @@ -42,7 +42,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -51,7 +50,7 @@ jobs: name: Python Flink IO runs-on: self-hosted strategy: - fail-fast: true + fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} @@ -61,7 +60,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Set python version run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV - name: Setup environment diff --git a/.github/workflows/job-precommit-python-gcp-io.yml b/.github/workflows/job-precommit-python-gcp-io.yml index 85b4640b2b92..005507e56b29 100644 --- a/.github/workflows/job-precommit-python-gcp-io.yml +++ b/.github/workflows/job-precommit-python-gcp-io.yml @@ -42,7 +42,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -51,7 +50,7 @@ jobs: name: Python GCP IO runs-on: self-hosted strategy: - fail-fast: true + fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} @@ -61,7 +60,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Set python version run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV - name: Setup environment diff --git a/.github/workflows/job-precommit-python-hadoop-io.yml b/.github/workflows/job-precommit-python-hadoop-io.yml index 82198d0b9073..5f403e17f971 100644 --- a/.github/workflows/job-precommit-python-hadoop-io.yml +++ b/.github/workflows/job-precommit-python-hadoop-io.yml @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -52,7 +51,7 @@ jobs: name: Python Hadoop IO runs-on: self-hosted strategy: - fail-fast: true + fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} @@ -62,7 +61,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Set python version run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV - name: Setup environment diff --git a/.github/workflows/job-precommit-python-ml.yml b/.github/workflows/job-precommit-python-ml.yml index ccf15768c9b2..d0d2b4a9d8c0 100644 --- a/.github/workflows/job-precommit-python-ml.yml +++ b/.github/workflows/job-precommit-python-ml.yml @@ -42,7 +42,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -51,7 +50,7 @@ jobs: name: Python ML runs-on: self-hosted strategy: - fail-fast: true + fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} @@ -61,7 +60,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Set python version run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV - name: Setup environment