From 8ce051e8181361cd7525235ae7a11f229fcb0cf6 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Fri, 9 Sep 2022 09:46:46 -0600 Subject: [PATCH 1/6] updating CI.md and adding yaml file --- .../job-precommit-python-dataframe.yml | 76 +++++++++++++++++++ CI.md | 8 ++ 2 files changed, 84 insertions(+) create mode 100644 .github/workflows/job-precommit-python-dataframe.yml diff --git a/.github/workflows/job-precommit-python-dataframe.yml b/.github/workflows/job-precommit-python-dataframe.yml new file mode 100644 index 000000000000..f427e3b92337 --- /dev/null +++ b/.github/workflows/job-precommit-python-dataframe.yml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +# Runs PreCommit Python Dataframe scheduled on cron, on push to master and on pull request + +name: PreCommit Python Dataframe + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + precommit_python_dataframe: + needs: set-properties + name: PreCommit Python Dataframe + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cloud + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cloud + arguments: "-Pposargs=apache_beam/dataframe/" + - name: Run :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cython + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cython + arguments: "-Pposargs=apache_beam/dataframe/" diff --git a/CI.md b/CI.md index ed75a4dd4433..4e56bcbacc49 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,14 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PreCommit Workflows + +#### Python PreCommit Dataframe - [job-precommit-python-dataframe.yml](.github/workflows/job-precommit-python-dataframe.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|---------------------------------|----------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Python PreCommit Dataframe | Runs Python PreCommit Dataframe | Yes | Yes | Yes | No | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From 61aee97fc7f66cea3b5a543d8c2f19ffa69df81c Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Wed, 14 Sep 2022 20:28:15 -0600 Subject: [PATCH 2/6] checkout version, permissions, pull_request_target, tox env --- .../job-precommit-python-dataframe.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/job-precommit-python-dataframe.yml b/.github/workflows/job-precommit-python-dataframe.yml index f427e3b92337..0575951570b4 100644 --- a/.github/workflows/job-precommit-python-dataframe.yml +++ b/.github/workflows/job-precommit-python-dataframe.yml @@ -28,9 +28,10 @@ on: push: branches: ['master', 'release-*'] tags: 'v*' - pull_request: + pull_request_target: branches: ['master', 'release-*'] tags: 'v*' +permissions: read-all jobs: set-properties: @@ -39,8 +40,9 @@ jobs: properties: ${{ steps.test-properties.outputs.properties }} steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - id: test-properties @@ -54,23 +56,20 @@ jobs: fail-fast: false matrix: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + tox-env: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.TOX_ENV}} steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - name: Set python version run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV - name: Setup environment uses: ./.github/actions/setup-self-hosted-action - - name: Run :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cloud + - name: Run :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}${{matrix.tox-env}} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cloud - arguments: "-Pposargs=apache_beam/dataframe/" - - name: Run :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cython - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cython + gradle-command: :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}${{matrix.tox-env}} arguments: "-Pposargs=apache_beam/dataframe/" From 759f8c12356803d8c8454c495ed4bd3d4835eafa Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Mon, 19 Sep 2022 13:27:46 -0600 Subject: [PATCH 3/6] Update job-precommit-python-dataframe.yml --- .github/workflows/job-precommit-python-dataframe.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/job-precommit-python-dataframe.yml b/.github/workflows/job-precommit-python-dataframe.yml index 0575951570b4..cfb3fe7bfdec 100644 --- a/.github/workflows/job-precommit-python-dataframe.yml +++ b/.github/workflows/job-precommit-python-dataframe.yml @@ -22,7 +22,6 @@ name: PreCommit Python Dataframe on: - workflow_dispatch: schedule: - cron: '0 */6 * * *' push: From 7ab05be112e7955d8ea95d82e6de64ce38830780 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Fri, 23 Sep 2022 12:36:07 -0600 Subject: [PATCH 4/6] adding ubuntu to runs-on --- .github/workflows/job-precommit-python-dataframe.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/job-precommit-python-dataframe.yml b/.github/workflows/job-precommit-python-dataframe.yml index cfb3fe7bfdec..c7da7f856f95 100644 --- a/.github/workflows/job-precommit-python-dataframe.yml +++ b/.github/workflows/job-precommit-python-dataframe.yml @@ -34,7 +34,7 @@ permissions: read-all jobs: set-properties: - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] outputs: properties: ${{ steps.test-properties.outputs.properties }} steps: @@ -50,7 +50,7 @@ jobs: precommit_python_dataframe: needs: set-properties name: PreCommit Python Dataframe - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] strategy: fail-fast: false matrix: From 60bcd22ed923cade9a6a9ee8c52c8bb88c8c262a Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Mon, 26 Sep 2022 14:01:28 -0600 Subject: [PATCH 5/6] adding workflow_dispatch and updating ci.md --- .../job-precommit-python-dataframe.yml | 2 ++ CI.md | 20 ++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/job-precommit-python-dataframe.yml b/.github/workflows/job-precommit-python-dataframe.yml index c7da7f856f95..44fd21059daf 100644 --- a/.github/workflows/job-precommit-python-dataframe.yml +++ b/.github/workflows/job-precommit-python-dataframe.yml @@ -22,6 +22,7 @@ name: PreCommit Python Dataframe on: + workflow_dispatch: schedule: - cron: '0 */6 * * *' push: @@ -30,6 +31,7 @@ on: pull_request_target: branches: ['master', 'release-*'] tags: 'v*' + paths: [ 'sdks/python/apache_beam/dataframe/**'] permissions: read-all jobs: diff --git a/CI.md b/CI.md index 4e56bcbacc49..7d593a4e3571 100644 --- a/CI.md +++ b/CI.md @@ -125,16 +125,26 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | -### PreCommit Workflows +### All migrated workflows run based on the following triggers + +| Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Workflow Dispatch | +|-------------|------------------|-----------------------|---------------|-------------------| +| PostCommit | No | Yes | Yes | Yes | +| PreCommit | Yes | Yes | Yes | Yes | -#### Python PreCommit Dataframe - [job-precommit-python-dataframe.yml](.github/workflows/job-precommit-python-dataframe.yml) +### PreCommit Workflows +| Workflow | Description | Requires GCP Credentials | +|--------------------------------------------------------------------------------------------|---------------------------------|--------------------------| +| [job-precommit-python-dataframe.yml](.github/workflows/job-precommit-python-dataframe.yml) | Runs Python PreCommit Dataframe | No | -| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | -|---------------------------------|----------------------------------|------------------|-----------------------|---------------|--------------------------| -| Run Python PreCommit Dataframe | Runs Python PreCommit Dataframe | Yes | Yes | Yes | No | +### PostCommit Workflows +| Workflow | Description | Requires GCP Credentials | +|------------------------------------------------------------------------------------|-------------------------|--------------------------| +| [job-postcommit-placeholder.yml](.github/workflows/job-postcommit-placeholder.yml) | Description placeholder | Yes/No | ### GitHub Action Tips +* All migrated workflows get executed on **pre-configured self-hosted** runners. For this reason, GCP credentials are **only** needed when running the workflows in a different runner. * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. In this case please attach link to the modified workflow run executed on your fork. * Possible timeouts with macOS runner - existing issue: [(X) This check failed - sometimes happens on macOS runner #841](https://github.com/actions/virtual-environments/issues/841) From a084fb1f5db2dd28235fe5f4ae8be397f828db74 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Thu, 29 Sep 2022 12:59:29 -0600 Subject: [PATCH 6/6] Update job-precommit-python-dataframe.yml --- .github/workflows/job-precommit-python-dataframe.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/job-precommit-python-dataframe.yml b/.github/workflows/job-precommit-python-dataframe.yml index 44fd21059daf..f2795424cfed 100644 --- a/.github/workflows/job-precommit-python-dataframe.yml +++ b/.github/workflows/job-precommit-python-dataframe.yml @@ -45,7 +45,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive + submodules: false - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -64,7 +64,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive + submodules: false - name: Set python version run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV - name: Setup environment