From 492b45077036ed3e3dfcdc5005532a75cca3097f Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Tue, 20 Sep 2022 15:33:46 -0500 Subject: [PATCH 1/5] add runner portability python flink --- ...ommit-python-runners-portability-flink.yml | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 .github/workflows/job-precommit-python-runners-portability-flink.yml diff --git a/.github/workflows/job-precommit-python-runners-portability-flink.yml b/.github/workflows/job-precommit-python-runners-portability-flink.yml new file mode 100644 index 000000000000..6cf4fdb0abbd --- /dev/null +++ b/.github/workflows/job-precommit-python-runners-portability-flink.yml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Test for Python Precommit Runners Portability Flink + + +name: Python Precommit Runners Portability Flink +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: ['v*'] + pull_request: + branches: ['master', 'release-*'] + tags: ['v*'] + #paths: ['sdks/python/apache_beam/examples/**', + # 'sdks/python/apache_beam/examples/**'] +permissions: read-all + +env: + tests: "'apache_beam/runners/portability/flink_runner.py \ + apache_beam/runners/portability/flink_runner_test.py \ + apache_beam/runners/portability/flink_uber_jar_job_server.py \ + apache_beam/runners/portability/flink_uber_jar_job_server_test.py'" + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + runners_portability_flink: + needs: set-properties + name: Python Runners Portability Flink + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + persist-credentials: false + submodules: recursive + - name: Set python version and tox env + run: | + echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:python:test-suites:tox:${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cloud + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cloud + arguments: "-Pposargs=${{env.tests}}" + - name: Run :sdks:python:test-suites:tox:y${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cython + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:python:test-suites:tox:py${{ env.PYTHON_VERSION }}:testPy${{ env.PYTHON_VERSION }}Cython + arguments: "-Pposargs=${{env.tests}}" \ No newline at end of file From 697fe9fdbb64f1bf084f9f36ea7d37c115dc427c Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Fri, 23 Sep 2022 13:07:05 -0500 Subject: [PATCH 2/5] update job with required fixes --- ...b-precommit-python-runners-portability-flink.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/job-precommit-python-runners-portability-flink.yml b/.github/workflows/job-precommit-python-runners-portability-flink.yml index 6cf4fdb0abbd..fa1a39d1345c 100644 --- a/.github/workflows/job-precommit-python-runners-portability-flink.yml +++ b/.github/workflows/job-precommit-python-runners-portability-flink.yml @@ -20,17 +20,18 @@ name: Python Precommit Runners Portability Flink on: - workflow_dispatch: schedule: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] tags: ['v*'] - pull_request: + pull_request_target: branches: ['master', 'release-*'] tags: ['v*'] - #paths: ['sdks/python/apache_beam/examples/**', - # 'sdks/python/apache_beam/examples/**'] + paths: ['sdks/python/apache_beam/runners/portability/flink_runner.py', + 'sdks/python/apache_beam/runners/portability/flink_runner_test.py', + 'sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server.py', + 'sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server_test.py'] permissions: read-all env: @@ -41,13 +42,14 @@ env: jobs: set-properties: - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] outputs: properties: ${{ steps.test-properties.outputs.properties }} steps: - name: Checkout code uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - id: test-properties @@ -65,6 +67,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - name: Set python version and tox env From d6196eaf70f542dc7f7a8e2a70a41070c467feca Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Wed, 28 Sep 2022 08:50:20 -0500 Subject: [PATCH 3/5] resolve CI conflicts and fix --- CI.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI.md b/CI.md index ee911584f7f0..4b43956e24b5 100644 --- a/CI.md +++ b/CI.md @@ -135,7 +135,7 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog ### PreCommit Workflows | Workflow | Description | Requires GCP Credentials | |----------------------------------------------------------------------------------|-------------------------|---------------------------| -| [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | +| [job-precommit-python-runners-portability-flink.yml](.github/workflows/job-precommit-python-runners-portability-flink.yml) | Run Python Precommit Runners Portability Flink | Yes/No | ### PostCommit Workflows | Workflow | Description | Requires GCP Credentials | From 5117b3a76d3c47576da7c1092c70f43d540c65fc Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Wed, 28 Sep 2022 08:53:19 -0500 Subject: [PATCH 4/5] add paths to push trigger in workflow --- .../job-precommit-python-runners-portability-flink.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/job-precommit-python-runners-portability-flink.yml b/.github/workflows/job-precommit-python-runners-portability-flink.yml index fa1a39d1345c..346515e7a8bf 100644 --- a/.github/workflows/job-precommit-python-runners-portability-flink.yml +++ b/.github/workflows/job-precommit-python-runners-portability-flink.yml @@ -25,6 +25,10 @@ on: push: branches: ['master', 'release-*'] tags: ['v*'] + paths: [ 'sdks/python/apache_beam/runners/portability/flink_runner.py', + 'sdks/python/apache_beam/runners/portability/flink_runner_test.py', + 'sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server.py', + 'sdks/python/apache_beam/runners/portability/flink_uber_jar_job_server_test.py' ] pull_request_target: branches: ['master', 'release-*'] tags: ['v*'] From 2b508720ee23c5c3b50e91ff319c49cfe10d82a5 Mon Sep 17 00:00:00 2001 From: Marco Robles Date: Sat, 1 Oct 2022 10:16:26 -0500 Subject: [PATCH 5/5] update yml job file --- .../job-precommit-python-runners-portability-flink.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/job-precommit-python-runners-portability-flink.yml b/.github/workflows/job-precommit-python-runners-portability-flink.yml index 346515e7a8bf..15c2124bf642 100644 --- a/.github/workflows/job-precommit-python-runners-portability-flink.yml +++ b/.github/workflows/job-precommit-python-runners-portability-flink.yml @@ -55,7 +55,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -73,7 +72,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Set python version and tox env run: | echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV