From 284223968e6d7bc619939139f4f29a060dad57b4 Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Wed, 21 Sep 2022 15:42:58 -0600 Subject: [PATCH 1/6] Adding yaml file and needed changes --- .../job-postcommit-beam-metrics-publish.yml | 52 +++++++++++++++++++ .test-infra/jenkins/README.md | 1 - .../job_PostCommit_BeamMetrics_Publish.groovy | 41 --------------- .test-infra/metrics/build.gradle | 6 +-- .../metrics/build_and_publish_containers.sh | 12 ++--- CI.md | 8 +++ 6 files changed, 69 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/job-postcommit-beam-metrics-publish.yml delete mode 100644 .test-infra/jenkins/job_PostCommit_BeamMetrics_Publish.groovy diff --git a/.github/workflows/job-postcommit-beam-metrics-publish.yml b/.github/workflows/job-postcommit-beam-metrics-publish.yml new file mode 100644 index 000000000000..3197f737a034 --- /dev/null +++ b/.github/workflows/job-postcommit-beam-metrics-publish.yml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +# Applies new configuration to Beam Metrics infrastructure. + +name: PostCommit Beam Metrics Publish + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master'] +permissions: read-all + +jobs: + postcommit-beam-metrics-publish: + name: Run PostCommit Beam Metrics Publish + runs-on: self-hosted + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Install kubectl + uses: azure/setup-kubectl@v3 + - name: Get Credentials + run: | + gcloud container clusters get-credentials metrics-migration-test --zone us-central1-c --project apache-beam-testing + - name: Run :beam-test-infra-metrics:deploy + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :beam-test-infra-metrics:deploy diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 08b9665a92ba..7c923e69a96a 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -61,7 +61,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | Name | Link | PR Trigger Phrase | Cron Status | |------|------|-------------------|-------------| -| beam_PostCommit_BeamMetrics_Publish | [cron](https://ci-beam.apache.org/job/beam_PostCommit_BeamMetrics_Publish/) | N/A | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_BeamMetrics_Publish/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_BeamMetrics_Publish) | | beam_PostCommit_XVR_Flink | [cron](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Flink/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Flink_PR/) | `Run XVR_Flink PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Flink/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Flink) | | beam_PostCommit_XVR_Samza | [cron](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Samza/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Samza_PR/) | `Run XVR_Samza PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Samza/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Samza) | | beam_PostCommit_XVR_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Spark_PR/) | `Run XVR_Spark PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_XVR_Spark) | diff --git a/.test-infra/jenkins/job_PostCommit_BeamMetrics_Publish.groovy b/.test-infra/jenkins/job_PostCommit_BeamMetrics_Publish.groovy deleted file mode 100644 index 34912d2825f1..000000000000 --- a/.test-infra/jenkins/job_PostCommit_BeamMetrics_Publish.groovy +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, eit her express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import Kubernetes -import PostcommitJobBuilder - -PostcommitJobBuilder.postCommitJob('beam_PostCommit_BeamMetrics_Publish', 'Run Beam Metrics deployment', - 'Beam Metrics deployment', this) { - - description('Applies new configuration to Beam Metrics infrastructure.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - - String kubeconfig = commonJobProperties.getKubeconfigLocationForNamespace('default') - Kubernetes.create(delegate, kubeconfig, '', 'metrics') - - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':beam-test-infra-metrics:deploy') - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/.test-infra/metrics/build.gradle b/.test-infra/metrics/build.gradle index 679ecd35735d..450331c1d8d0 100644 --- a/.test-infra/metrics/build.gradle +++ b/.test-infra/metrics/build.gradle @@ -123,9 +123,9 @@ task deploy { exec { executable 'sh' args '-c', "kubectl set image deployment/beamgrafana \ - beamgrafana=gcr.io/apache-beam-testing/beamgrafana:${project.lastCommitId} \ - beammetricssyncjenkins=gcr.io/apache-beam-testing/beammetricssyncjenkins:${project.lastCommitId} \ - beammetricssyncgithub=gcr.io/apache-beam-testing/beammetricssyncgithub:${project.lastCommitId}" + beamgrafana-test=gcr.io/apache-beam-testing/beamgrafana-test:${project.lastCommitId} \ + beammetricssyncjenkins-test=gcr.io/apache-beam-testing/beammetricssyncjenkins-test:${project.lastCommitId} \ + beammetricssyncgithub-test=gcr.io/apache-beam-testing/beammetricssyncgithub-test:${project.lastCommitId}" } } else { println 'Nothing to do, exiting.' diff --git a/.test-infra/metrics/build_and_publish_containers.sh b/.test-infra/metrics/build_and_publish_containers.sh index dbc3dbbfe66d..a6ea3d175e45 100755 --- a/.test-infra/metrics/build_and_publish_containers.sh +++ b/.test-infra/metrics/build_and_publish_containers.sh @@ -38,14 +38,14 @@ CONTAINER_VERSION_NAME=${2:-latest} echo echo ===========Building containers========== -docker build -t gcr.io/${PROJECT_ID}/beamgrafana:$CONTAINER_VERSION_NAME ./grafana -docker build -t gcr.io/${PROJECT_ID}/beammetricssyncjenkins:$CONTAINER_VERSION_NAME ./sync/jenkins -docker build -t gcr.io/${PROJECT_ID}/beammetricssyncgithub:$CONTAINER_VERSION_NAME ./sync/github +docker build -t gcr.io/${PROJECT_ID}/beamgrafana-test:$CONTAINER_VERSION_NAME ./grafana +docker build -t gcr.io/${PROJECT_ID}/beammetricssyncjenkins-test:$CONTAINER_VERSION_NAME ./sync/jenkins +docker build -t gcr.io/${PROJECT_ID}/beammetricssyncgithub-test:$CONTAINER_VERSION_NAME ./sync/github if [ "$DO_PUSH" = true ]; then echo echo ===========Publishing containers========== - docker push gcr.io/${PROJECT_ID}/beamgrafana:$CONTAINER_VERSION_NAME - docker push gcr.io/${PROJECT_ID}/beammetricssyncjenkins:$CONTAINER_VERSION_NAME - docker push gcr.io/${PROJECT_ID}/beammetricssyncgithub:$CONTAINER_VERSION_NAME + docker push gcr.io/${PROJECT_ID}/beamgrafana-test:$CONTAINER_VERSION_NAME + docker push gcr.io/${PROJECT_ID}/beammetricssyncjenkins-test:$CONTAINER_VERSION_NAME + docker push gcr.io/${PROJECT_ID}/beammetricssyncgithub-test:$CONTAINER_VERSION_NAME fi diff --git a/CI.md b/CI.md index ed75a4dd4433..94c286fc7e7e 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,14 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PostCommit Workflows + +#### PostCommit Beam Metrics Publish - [job-postcommit-beam-metrics-publish.yml](.github/workflows/job-postcommit-beam-metrics-publish.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-------------------------------------|----------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run PostCommit Beam Metrics Publish | Applies new configuration to Beam Metrics infrastructure | No | Yes | Yes | Yes | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From a9a788657f111265cbfdcd379e6e822ddab83116 Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Thu, 22 Sep 2022 10:09:20 -0600 Subject: [PATCH 2/6] Update job-postcommit-beam-metrics-publish.yml --- .github/workflows/job-postcommit-beam-metrics-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/job-postcommit-beam-metrics-publish.yml b/.github/workflows/job-postcommit-beam-metrics-publish.yml index 3197f737a034..7e078867dc63 100644 --- a/.github/workflows/job-postcommit-beam-metrics-publish.yml +++ b/.github/workflows/job-postcommit-beam-metrics-publish.yml @@ -43,6 +43,9 @@ jobs: uses: ./.github/actions/setup-self-hosted-action - name: Install kubectl uses: azure/setup-kubectl@v3 + - name: Configure Docker + run: | + gcloud auth configure-docker - name: Get Credentials run: | gcloud container clusters get-credentials metrics-migration-test --zone us-central1-c --project apache-beam-testing From eec06a4b58946ec4a01793eff9a96b87732c2859 Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Fri, 23 Sep 2022 12:12:09 -0600 Subject: [PATCH 3/6] Update job-postcommit-beam-metrics-publish.yml --- .github/workflows/job-postcommit-beam-metrics-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/job-postcommit-beam-metrics-publish.yml b/.github/workflows/job-postcommit-beam-metrics-publish.yml index 7e078867dc63..937aa2552633 100644 --- a/.github/workflows/job-postcommit-beam-metrics-publish.yml +++ b/.github/workflows/job-postcommit-beam-metrics-publish.yml @@ -32,7 +32,7 @@ permissions: read-all jobs: postcommit-beam-metrics-publish: name: Run PostCommit Beam Metrics Publish - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] steps: - name: Checkout code uses: actions/checkout@v3 From dd3f908e8d123f368b9addb6a7b26602443c521a Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Mon, 26 Sep 2022 12:40:31 -0600 Subject: [PATCH 4/6] Update CI.md --- CI.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/CI.md b/CI.md index 94c286fc7e7e..c20a471ea665 100644 --- a/CI.md +++ b/CI.md @@ -125,13 +125,22 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | -### PostCommit Workflows +### All migrated workflows run based on the following triggers + +| Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Workflow Dispatch | +|-------------|------------------|-----------------------|---------------|-------------------| +| PostCommit | No | Yes | Yes | Yes | +| PreCommit | Yes | Yes | Yes | Yes | -#### PostCommit Beam Metrics Publish - [job-postcommit-beam-metrics-publish.yml](.github/workflows/job-postcommit-beam-metrics-publish.yml) +### PreCommit Workflows +| Workflow | Description | Requires GCP Credentials | +|----------------------------------------------------------------------------------|-------------------------|---------------------------| +| [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | -| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | -|-------------------------------------|----------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| -| Run PostCommit Beam Metrics Publish | Applies new configuration to Beam Metrics infrastructure | No | Yes | Yes | Yes | +### PostCommit Workflows +| Workflow | Description | Requires GCP Credentials | +|------------------------------------------------------------------------------------------------------|----------------------------------------------------------|--------------------------| +| [job-postcommit-beam-metrics-publish.yml](.github/workflows/job-postcommit-beam-metrics-publish.yml) | Applies new configuration to Beam Metrics infrastructure | Yes | ### GitHub Action Tips From ad541d8592c670263468bd98bedebebbffe56b8e Mon Sep 17 00:00:00 2001 From: Fernando Morales <80284146+fernando-wizeline@users.noreply.github.com> Date: Mon, 26 Sep 2022 13:20:13 -0600 Subject: [PATCH 5/6] Update CI.md --- CI.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI.md b/CI.md index c20a471ea665..cef80d1010dc 100644 --- a/CI.md +++ b/CI.md @@ -130,7 +130,7 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Workflow Dispatch | |-------------|------------------|-----------------------|---------------|-------------------| | PostCommit | No | Yes | Yes | Yes | -| PreCommit | Yes | Yes | Yes | Yes | +| PreCommit | Yes | Yes | Yes | Yes | ### PreCommit Workflows | Workflow | Description | Requires GCP Credentials | From 04d3a50239d4af71f3a956c7a46c22a46973a27a Mon Sep 17 00:00:00 2001 From: Fernando Morales Date: Thu, 29 Sep 2022 13:37:05 -0600 Subject: [PATCH 6/6] targeting original gcr images; submodules to false --- .../job-postcommit-beam-metrics-publish.yml | 2 +- .test-infra/metrics/build.gradle | 6 +++--- .test-infra/metrics/build_and_publish_containers.sh | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/job-postcommit-beam-metrics-publish.yml b/.github/workflows/job-postcommit-beam-metrics-publish.yml index 937aa2552633..1cbbae1d52d7 100644 --- a/.github/workflows/job-postcommit-beam-metrics-publish.yml +++ b/.github/workflows/job-postcommit-beam-metrics-publish.yml @@ -38,7 +38,7 @@ jobs: uses: actions/checkout@v3 with: persist-credentials: false - submodules: recursive + submodules: false - name: Setup environment uses: ./.github/actions/setup-self-hosted-action - name: Install kubectl diff --git a/.test-infra/metrics/build.gradle b/.test-infra/metrics/build.gradle index 450331c1d8d0..679ecd35735d 100644 --- a/.test-infra/metrics/build.gradle +++ b/.test-infra/metrics/build.gradle @@ -123,9 +123,9 @@ task deploy { exec { executable 'sh' args '-c', "kubectl set image deployment/beamgrafana \ - beamgrafana-test=gcr.io/apache-beam-testing/beamgrafana-test:${project.lastCommitId} \ - beammetricssyncjenkins-test=gcr.io/apache-beam-testing/beammetricssyncjenkins-test:${project.lastCommitId} \ - beammetricssyncgithub-test=gcr.io/apache-beam-testing/beammetricssyncgithub-test:${project.lastCommitId}" + beamgrafana=gcr.io/apache-beam-testing/beamgrafana:${project.lastCommitId} \ + beammetricssyncjenkins=gcr.io/apache-beam-testing/beammetricssyncjenkins:${project.lastCommitId} \ + beammetricssyncgithub=gcr.io/apache-beam-testing/beammetricssyncgithub:${project.lastCommitId}" } } else { println 'Nothing to do, exiting.' diff --git a/.test-infra/metrics/build_and_publish_containers.sh b/.test-infra/metrics/build_and_publish_containers.sh index a6ea3d175e45..dbc3dbbfe66d 100755 --- a/.test-infra/metrics/build_and_publish_containers.sh +++ b/.test-infra/metrics/build_and_publish_containers.sh @@ -38,14 +38,14 @@ CONTAINER_VERSION_NAME=${2:-latest} echo echo ===========Building containers========== -docker build -t gcr.io/${PROJECT_ID}/beamgrafana-test:$CONTAINER_VERSION_NAME ./grafana -docker build -t gcr.io/${PROJECT_ID}/beammetricssyncjenkins-test:$CONTAINER_VERSION_NAME ./sync/jenkins -docker build -t gcr.io/${PROJECT_ID}/beammetricssyncgithub-test:$CONTAINER_VERSION_NAME ./sync/github +docker build -t gcr.io/${PROJECT_ID}/beamgrafana:$CONTAINER_VERSION_NAME ./grafana +docker build -t gcr.io/${PROJECT_ID}/beammetricssyncjenkins:$CONTAINER_VERSION_NAME ./sync/jenkins +docker build -t gcr.io/${PROJECT_ID}/beammetricssyncgithub:$CONTAINER_VERSION_NAME ./sync/github if [ "$DO_PUSH" = true ]; then echo echo ===========Publishing containers========== - docker push gcr.io/${PROJECT_ID}/beamgrafana-test:$CONTAINER_VERSION_NAME - docker push gcr.io/${PROJECT_ID}/beammetricssyncjenkins-test:$CONTAINER_VERSION_NAME - docker push gcr.io/${PROJECT_ID}/beammetricssyncgithub-test:$CONTAINER_VERSION_NAME + docker push gcr.io/${PROJECT_ID}/beamgrafana:$CONTAINER_VERSION_NAME + docker push gcr.io/${PROJECT_ID}/beammetricssyncjenkins:$CONTAINER_VERSION_NAME + docker push gcr.io/${PROJECT_ID}/beammetricssyncgithub:$CONTAINER_VERSION_NAME fi