From b73f5f70ac4184b56a0a03922731c5f2f69b9566 Mon Sep 17 00:00:00 2001 From: Sergey Kalinin <91209855+snkalinin@users.noreply.github.com> Date: Tue, 16 Nov 2021 14:59:11 +0300 Subject: [PATCH 1/3] [Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83) * Update workflows for playground * Attempt to fix tests * Remove continue on error to catch errors * Fix linter problem for backend dockerfile * Update folder to run backend go linter * Moved flutter test to execution via gradle tasks --- .../workflows/build_playground_backend.yml | 14 ++++++++----- .../workflows/build_playground_frontend.yml | 20 +++++++------------ playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index 96e8aae9b0d5..bd2868768246 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,10 +17,8 @@ name: Build And Deploy Playground Backend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -45,12 +43,14 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint - run: "golangci-lint run internal/..." +# run: "golangci-lint run internal/..." + run: "golangci-lint run cmd/server/..." working-directory: playground/backend/ - continue-on-error: true + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue + run: rm ~/.m2/settings.xml - name: Run Tests run: ./gradlew playground:backend:test - continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,12 +62,16 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index 9c2d56a52428..f93e330442b0 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,12 +17,10 @@ name: Build And Deploy Playground Frontend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/backend/**'] + paths: ['playground/frontend/**'] workflow_dispatch: jobs: @@ -43,18 +41,10 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' - - name: Prepare Flutter lint - run: "flutter pub add flutter_lints --dev" - working-directory: playground/frontend/ - continue-on-error: true - name: Run Lint - run: "flutter analyze" - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:analyze - name: Run Tests - run: flutter test - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:test - name: install npm uses: actions/setup-node@v2 with: @@ -66,12 +56,16 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index d7275020dcfa..8846368ff8b6 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE +ARG BASE_IMAGE=apache/beam_java8_sdk:latest FROM golang:1.17-buster AS build # Setup Go Environment From d449275b8ebd623760b34981f56c8f62b140ad9d Mon Sep 17 00:00:00 2001 From: Ilya Date: Tue, 16 Nov 2021 15:07:49 +0300 Subject: [PATCH 2/3] Revert "[Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83)" (#88) This reverts commit b73f5f70ac4184b56a0a03922731c5f2f69b9566. --- .../workflows/build_playground_backend.yml | 14 +++++-------- .../workflows/build_playground_frontend.yml | 20 ++++++++++++------- playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index bd2868768246..96e8aae9b0d5 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,8 +17,10 @@ name: Build And Deploy Playground Backend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -43,14 +45,12 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint -# run: "golangci-lint run internal/..." - run: "golangci-lint run cmd/server/..." + run: "golangci-lint run internal/..." working-directory: playground/backend/ - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: rm ~/.m2/settings.xml + continue-on-error: true - name: Run Tests run: ./gradlew playground:backend:test + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,16 +62,12 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index f93e330442b0..9c2d56a52428 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,10 +17,12 @@ name: Build And Deploy Playground Frontend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/frontend/**'] + paths: ['playground/backend/**'] workflow_dispatch: jobs: @@ -41,10 +43,18 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' + - name: Prepare Flutter lint + run: "flutter pub add flutter_lints --dev" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Lint - run: ./gradlew playground:frontend:analyze + run: "flutter analyze" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Tests - run: ./gradlew playground:frontend:test + run: flutter test + working-directory: playground/frontend/ + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -56,16 +66,12 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index 8846368ff8b6..d7275020dcfa 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE=apache/beam_java8_sdk:latest +ARG BASE_IMAGE FROM golang:1.17-buster AS build # Setup Go Environment From bd4b0f951fcfe64f89ac7235b1f8355a9549e885 Mon Sep 17 00:00:00 2001 From: Pavel Avilov Date: Tue, 23 Nov 2021 17:10:36 +0300 Subject: [PATCH 3/3] Implement an example verification method; Setup logger; --- playground/infrastructure/ci_cd.py | 7 +++-- playground/infrastructure/ci_helper.py | 36 +++++++++++++++++++--- playground/infrastructure/config.py | 4 +++ playground/infrastructure/logger.py | 42 ++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 playground/infrastructure/logger.py diff --git a/playground/infrastructure/ci_cd.py b/playground/infrastructure/ci_cd.py index 23312c63878b..1400cc951345 100644 --- a/playground/infrastructure/ci_cd.py +++ b/playground/infrastructure/ci_cd.py @@ -12,28 +12,31 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import asyncio import os from cd_helper import CDHelper from ci_helper import CIHelper from helper import find_examples +from logger import setup_logger def ci_step(): """ CI step to verify all beam examples/tests/katas """ + setup_logger() root_dir = os.getenv("BEAM_ROOT_DIR") ci_helper = CIHelper() examples = find_examples(root_dir) - ci_helper.verify_examples(examples) + asyncio.run(ci_helper.verify_examples(examples)) def cd_step(): """ CD step to save all beam examples/tests/katas and their outputs on the Google Cloud """ + setup_logger() root_dir = os.getenv("BEAM_ROOT_DIR") cd_helper = CDHelper() examples = find_examples(root_dir) diff --git a/playground/infrastructure/ci_helper.py b/playground/infrastructure/ci_helper.py index 2822b732f6e8..773040ae35ee 100644 --- a/playground/infrastructure/ci_helper.py +++ b/playground/infrastructure/ci_helper.py @@ -13,7 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from typing import List + +from api.v1.api_pb2 import STATUS_COMPILE_ERROR, STATUS_ERROR, STATUS_RUN_ERROR, STATUS_RUN_TIMEOUT, \ + STATUS_VALIDATION_ERROR, STATUS_PREPARATION_ERROR +from config import Config +from grpc_client import GRPCClient from helper import Example, get_statuses @@ -24,7 +30,7 @@ class CIHelper: It is used to find and verify correctness if beam examples/katas/tests. """ - def verify_examples(self, examples: List[Example]): + async def verify_examples(self, examples: List[Example]): """ Verify correctness of beam examples. @@ -33,9 +39,9 @@ def verify_examples(self, examples: List[Example]): 3. Run processing for all examples to verify examples' code. """ get_statuses(examples) - self._verify_examples_status(examples) + await self._verify_examples_status(examples) - def _verify_examples_status(self, examples: List[Example]): + async def _verify_examples_status(self, examples: List[Example]): """ Verify statuses of beam examples. @@ -47,5 +53,25 @@ def _verify_examples_status(self, examples: List[Example]): Args: examples: beam examples that should be verified """ - # TODO [BEAM-13256] Implement - pass + client = GRPCClient() + verify_failed = False + for example in examples: + if example.status not in Config.ERROR_STATUSES: + continue + if example.status == STATUS_VALIDATION_ERROR: + logging.error(f"Example: {example.pipeline_id} has validation error") + elif example.status == STATUS_PREPARATION_ERROR: + logging.error(f"Example: {example.pipeline_id} has preparation error") + elif example.status == STATUS_ERROR: + logging.error(f"Example: {example.pipeline_id} has error during setup run builder") + elif example.status == STATUS_RUN_TIMEOUT: + logging.error(f"Example: {example.pipeline_id} failed because of timeout") + elif example.status == STATUS_COMPILE_ERROR: + err = await client.get_compile_output(example.pipeline_id) + logging.error(f"Example: {example.pipeline_id} has compilation error: {err}") + elif example.status == STATUS_RUN_ERROR: + err = await client.get_run_error(example.pipeline_id) + logging.error(f"Example: {example.pipeline_id} has execution error: {err}") + verify_failed = True + if verify_failed: + raise Exception("CI step failed due to errors in the examples") diff --git a/playground/infrastructure/config.py b/playground/infrastructure/config.py index 33e1ed7bfbaf..79ad1ceaad80 100644 --- a/playground/infrastructure/config.py +++ b/playground/infrastructure/config.py @@ -15,6 +15,8 @@ import os from dataclasses import dataclass +from api.v1.api_pb2 import STATUS_VALIDATION_ERROR, STATUS_ERROR, STATUS_PREPARATION_ERROR, STATUS_COMPILE_ERROR, \ + STATUS_RUN_TIMEOUT, STATUS_RUN_ERROR from api.v1.api_pb2 import SDK_JAVA @@ -22,4 +24,6 @@ @dataclass(frozen=True) class Config: SERVER_ADDRESS = os.getenv("SERVER_ADDRESS", "localhost:8080") + ERROR_STATUSES = [STATUS_VALIDATION_ERROR, STATUS_ERROR, STATUS_PREPARATION_ERROR, STATUS_COMPILE_ERROR, + STATUS_RUN_TIMEOUT, STATUS_RUN_ERROR] SUPPORTED_SDK = {'java': SDK_JAVA} diff --git a/playground/infrastructure/logger.py b/playground/infrastructure/logger.py new file mode 100644 index 000000000000..f8c0568c45dd --- /dev/null +++ b/playground/infrastructure/logger.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +from logging import INFO, WARNING, ERROR, CRITICAL + + +def setup_logger(): + """ + Setup logging. + + Add 2 handler in root logger: + StreamHandler - for logs(INFO and WARNING levels) to the stdout + StreamHandler - for logs(ERROR and CRITICAL levels) to the stderr + """ + log = logging.getLogger() + log.setLevel(logging.INFO) + formatter = logging.Formatter('[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s') + + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.addFilter(lambda record: record.levelno in (INFO, WARNING)) + stdout_handler.setFormatter(formatter) + + stderr_handler = logging.StreamHandler(sys.stderr) + stderr_handler.addFilter(lambda record: record.levelno in (ERROR, CRITICAL)) + stderr_handler.setFormatter(formatter) + + log.addHandler(stdout_handler) + log.addHandler(stderr_handler)