From b73f5f70ac4184b56a0a03922731c5f2f69b9566 Mon Sep 17 00:00:00 2001 From: Sergey Kalinin <91209855+snkalinin@users.noreply.github.com> Date: Tue, 16 Nov 2021 14:59:11 +0300 Subject: [PATCH 1/3] [Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83) * Update workflows for playground * Attempt to fix tests * Remove continue on error to catch errors * Fix linter problem for backend dockerfile * Update folder to run backend go linter * Moved flutter test to execution via gradle tasks --- .../workflows/build_playground_backend.yml | 14 ++++++++----- .../workflows/build_playground_frontend.yml | 20 +++++++------------ playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index 96e8aae9b0d5..bd2868768246 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,10 +17,8 @@ name: Build And Deploy Playground Backend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -45,12 +43,14 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint - run: "golangci-lint run internal/..." +# run: "golangci-lint run internal/..." + run: "golangci-lint run cmd/server/..." working-directory: playground/backend/ - continue-on-error: true + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue + run: rm ~/.m2/settings.xml - name: Run Tests run: ./gradlew playground:backend:test - continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,12 +62,16 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index 9c2d56a52428..f93e330442b0 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,12 +17,10 @@ name: Build And Deploy Playground Frontend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/backend/**'] + paths: ['playground/frontend/**'] workflow_dispatch: jobs: @@ -43,18 +41,10 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' - - name: Prepare Flutter lint - run: "flutter pub add flutter_lints --dev" - working-directory: playground/frontend/ - continue-on-error: true - name: Run Lint - run: "flutter analyze" - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:analyze - name: Run Tests - run: flutter test - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:test - name: install npm uses: actions/setup-node@v2 with: @@ -66,12 +56,16 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index d7275020dcfa..8846368ff8b6 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE +ARG BASE_IMAGE=apache/beam_java8_sdk:latest FROM golang:1.17-buster AS build # Setup Go Environment From d449275b8ebd623760b34981f56c8f62b140ad9d Mon Sep 17 00:00:00 2001 From: Ilya Date: Tue, 16 Nov 2021 15:07:49 +0300 Subject: [PATCH 2/3] Revert "[Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83)" (#88) This reverts commit b73f5f70ac4184b56a0a03922731c5f2f69b9566. --- .../workflows/build_playground_backend.yml | 14 +++++-------- .../workflows/build_playground_frontend.yml | 20 ++++++++++++------- playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index bd2868768246..96e8aae9b0d5 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,8 +17,10 @@ name: Build And Deploy Playground Backend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -43,14 +45,12 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint -# run: "golangci-lint run internal/..." - run: "golangci-lint run cmd/server/..." + run: "golangci-lint run internal/..." working-directory: playground/backend/ - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: rm ~/.m2/settings.xml + continue-on-error: true - name: Run Tests run: ./gradlew playground:backend:test + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,16 +62,12 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index f93e330442b0..9c2d56a52428 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,10 +17,12 @@ name: Build And Deploy Playground Frontend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/frontend/**'] + paths: ['playground/backend/**'] workflow_dispatch: jobs: @@ -41,10 +43,18 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' + - name: Prepare Flutter lint + run: "flutter pub add flutter_lints --dev" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Lint - run: ./gradlew playground:frontend:analyze + run: "flutter analyze" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Tests - run: ./gradlew playground:frontend:test + run: flutter test + working-directory: playground/frontend/ + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -56,16 +66,12 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index 8846368ff8b6..d7275020dcfa 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE=apache/beam_java8_sdk:latest +ARG BASE_IMAGE FROM golang:1.17-buster AS build # Setup Go Environment From 370e77815b675da123fe18fdeaf1c564e0b4e567 Mon Sep 17 00:00:00 2001 From: AydarZaynutdinov Date: Thu, 25 Nov 2021 18:14:58 +0300 Subject: [PATCH 3/3] [BEAM-13267][Playground] implemented a method to send examples to the backend to verify them --- playground/categories.yaml | 28 ++ playground/infrastructure/api/v1/api_pb2.py | 309 ++++++++++++------ .../infrastructure/api/v1/api_pb2_grpc.py | 166 ++++++---- playground/infrastructure/ci_cd.py | 11 +- playground/infrastructure/ci_helper.py | 2 +- playground/infrastructure/config.py | 18 +- playground/infrastructure/grpc_client.py | 2 +- playground/infrastructure/helper.py | 246 +++++++++++++- playground/infrastructure/requirements.txt | 23 ++ playground/infrastructure/test_ci_cd.py | 18 +- playground/infrastructure/test_ci_helper.py | 6 +- playground/infrastructure/test_grpc_client.py | 2 +- playground/infrastructure/test_helper.py | 220 +++++++++++++ 13 files changed, 850 insertions(+), 201 deletions(-) create mode 100644 playground/categories.yaml create mode 100644 playground/infrastructure/requirements.txt create mode 100644 playground/infrastructure/test_helper.py diff --git a/playground/categories.yaml b/playground/categories.yaml new file mode 100644 index 000000000000..f3cfdd3ecd51 --- /dev/null +++ b/playground/categories.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +categories: + - Side Input + - Side Output + - Testing + - Schemas + - Transform + - IO + - Metrics + - Options + - Coders + - Stateful Processing diff --git a/playground/infrastructure/api/v1/api_pb2.py b/playground/infrastructure/api/v1/api_pb2.py index 11aab06d3d06..dd5aa0f0b470 100644 --- a/playground/infrastructure/api/v1/api_pb2.py +++ b/playground/infrastructure/api/v1/api_pb2.py @@ -33,7 +33,7 @@ syntax='proto3', serialized_options=b'Z6beam.apache.org/playground/backend/internal;playground', create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\tapi.proto\x12\x06\x61pi.v1\"8\n\x0eRunCodeRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x18\n\x03sdk\x18\x02 \x01(\x0e\x32\x0b.api.v1.Sdk\"(\n\x0fRunCodeResponse\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"+\n\x12\x43heckStatusRequest\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"5\n\x13\x43heckStatusResponse\x12\x1e\n\x06status\x18\x01 \x01(\x0e\x32\x0e.api.v1.Status\"0\n\x17GetCompileOutputRequest\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"V\n\x18GetCompileOutputResponse\x12\x0e\n\x06output\x18\x01 \x01(\t\x12*\n\x12\x63ompilation_status\x18\x02 \x01(\x0e\x32\x0e.api.v1.Status\",\n\x13GetRunOutputRequest\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"&\n\x14GetRunOutputResponse\x12\x0e\n\x06output\x18\x01 \x01(\t\"+\n\x12GetRunErrorRequest\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"%\n\x13GetRunErrorResponse\x12\x0e\n\x06output\x18\x01 \x01(\t\"F\n\x18GetListOfExamplesRequest\x12\x18\n\x03sdk\x18\x01 \x01(\x0e\x32\x0b.api.v1.Sdk\x12\x10\n\x08\x63\x61tegory\x18\x02 \x01(\t\"e\n\x07\x45xample\x12\x14\n\x0c\x65xample_uuid\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12!\n\x04type\x18\x04 \x01(\x0e\x32\x13.api.v1.ExampleType\"\x9d\x01\n\nCategories\x12\x18\n\x03sdk\x18\x01 \x01(\x0e\x32\x0b.api.v1.Sdk\x12/\n\ncategories\x18\x02 \x03(\x0b\x32\x1b.api.v1.Categories.Category\x1a\x44\n\x08\x43\x61tegory\x12\x15\n\rcategory_name\x18\x01 \x01(\t\x12!\n\x08\x65xamples\x18\x02 \x03(\x0b\x32\x0f.api.v1.Example\"E\n\x19GetListOfExamplesResponse\x12(\n\x0csdk_examples\x18\x01 \x03(\x0b\x32\x12.api.v1.Categories\")\n\x11GetExampleRequest\x12\x14\n\x0c\x65xample_uuid\x18\x01 \x01(\t\"\"\n\x12GetExampleResponse\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t*R\n\x03Sdk\x12\x13\n\x0fSDK_UNSPECIFIED\x10\x00\x12\x0c\n\x08SDK_JAVA\x10\x01\x12\n\n\x06SDK_GO\x10\x02\x12\x0e\n\nSDK_PYTHON\x10\x03\x12\x0c\n\x08SDK_SCIO\x10\x04*\xa3\x02\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x15\n\x11STATUS_VALIDATING\x10\x01\x12\x1b\n\x17STATUS_VALIDATION_ERROR\x10\x02\x12\x14\n\x10STATUS_PREPARING\x10\x03\x12\x1c\n\x18STATUS_PREPARATION_ERROR\x10\x04\x12\x14\n\x10STATUS_COMPILING\x10\x05\x12\x18\n\x14STATUS_COMPILE_ERROR\x10\x06\x12\x14\n\x10STATUS_EXECUTING\x10\x07\x12\x13\n\x0fSTATUS_FINISHED\x10\x08\x12\x14\n\x10STATUS_RUN_ERROR\x10\t\x12\x10\n\x0cSTATUS_ERROR\x10\n\x12\x16\n\x12STATUS_RUN_TIMEOUT\x10\x0b*Z\n\x0b\x45xampleType\x12\x18\n\x14\x45XAMPLE_TYPE_DEFAULT\x10\x00\x12\x15\n\x11\x45XAMPLE_TYPE_KATA\x10\x01\x12\x1a\n\x16\x45XAMPLE_TYPE_UNIT_TEST\x10\x02\x32\xed\x04\n\x11PlaygroundService\x12:\n\x07RunCode\x12\x16.api.v1.RunCodeRequest\x1a\x17.api.v1.RunCodeResponse\x12\x46\n\x0b\x43heckStatus\x12\x1a.api.v1.CheckStatusRequest\x1a\x1b.api.v1.CheckStatusResponse\x12I\n\x0cGetRunOutput\x12\x1b.api.v1.GetRunOutputRequest\x1a\x1c.api.v1.GetRunOutputResponse\x12\x46\n\x0bGetRunError\x12\x1a.api.v1.GetRunErrorRequest\x1a\x1b.api.v1.GetRunErrorResponse\x12U\n\x10GetCompileOutput\x12\x1f.api.v1.GetCompileOutputRequest\x1a .api.v1.GetCompileOutputResponse\x12X\n\x11GetListOfExamples\x12 .api.v1.GetListOfExamplesRequest\x1a!.api.v1.GetListOfExamplesResponse\x12\x43\n\nGetExample\x12\x19.api.v1.GetExampleRequest\x1a\x1a.api.v1.GetExampleResponse\x12K\n\x10GetExampleOutput\x12\x19.api.v1.GetExampleRequest\x1a\x1c.api.v1.GetRunOutputResponseB8Z6beam.apache.org/playground/backend/internal;playgroundb\x06proto3' + serialized_pb=b'\n\tapi.proto\x12\x06\x61pi.v1\"8\n\x0eRunCodeRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x18\n\x03sdk\x18\x02 \x01(\x0e\x32\x0b.api.v1.Sdk\"(\n\x0fRunCodeResponse\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"+\n\x12\x43heckStatusRequest\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"5\n\x13\x43heckStatusResponse\x12\x1e\n\x06status\x18\x01 \x01(\x0e\x32\x0e.api.v1.Status\"0\n\x17GetCompileOutputRequest\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"V\n\x18GetCompileOutputResponse\x12\x0e\n\x06output\x18\x01 \x01(\t\x12*\n\x12\x63ompilation_status\x18\x02 \x01(\x0e\x32\x0e.api.v1.Status\",\n\x13GetRunOutputRequest\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"&\n\x14GetRunOutputResponse\x12\x0e\n\x06output\x18\x01 \x01(\t\"+\n\x12GetRunErrorRequest\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"%\n\x13GetRunErrorResponse\x12\x0e\n\x06output\x18\x01 \x01(\t\"&\n\rCancelRequest\x12\x15\n\rpipeline_uuid\x18\x01 \x01(\t\"\x10\n\x0e\x43\x61ncelResponse\"J\n\x1cGetPrecompiledObjectsRequest\x12\x18\n\x03sdk\x18\x01 \x01(\x0e\x32\x0b.api.v1.Sdk\x12\x10\n\x08\x63\x61tegory\x18\x02 \x01(\t\"w\n\x11PrecompiledObject\x12\x12\n\ncloud_path\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12+\n\x04type\x18\x04 \x01(\x0e\x32\x1d.api.v1.PrecompiledObjectType\"\xb2\x01\n\nCategories\x12\x18\n\x03sdk\x18\x01 \x01(\x0e\x32\x0b.api.v1.Sdk\x12/\n\ncategories\x18\x02 \x03(\x0b\x32\x1b.api.v1.Categories.Category\x1aY\n\x08\x43\x61tegory\x12\x15\n\rcategory_name\x18\x01 \x01(\t\x12\x36\n\x13precompiled_objects\x18\x02 \x03(\x0b\x32\x19.api.v1.PrecompiledObject\"K\n\x1dGetPrecompiledObjectsResponse\x12*\n\x0esdk_categories\x18\x01 \x03(\x0b\x32\x12.api.v1.Categories\"1\n\x1bGetPrecompiledObjectRequest\x12\x12\n\ncloud_path\x18\x01 \x01(\t\"0\n GetPrecompiledObjectCodeResponse\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t*R\n\x03Sdk\x12\x13\n\x0fSDK_UNSPECIFIED\x10\x00\x12\x0c\n\x08SDK_JAVA\x10\x01\x12\n\n\x06SDK_GO\x10\x02\x12\x0e\n\nSDK_PYTHON\x10\x03\x12\x0c\n\x08SDK_SCIO\x10\x04*\xb8\x02\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x15\n\x11STATUS_VALIDATING\x10\x01\x12\x1b\n\x17STATUS_VALIDATION_ERROR\x10\x02\x12\x14\n\x10STATUS_PREPARING\x10\x03\x12\x1c\n\x18STATUS_PREPARATION_ERROR\x10\x04\x12\x14\n\x10STATUS_COMPILING\x10\x05\x12\x18\n\x14STATUS_COMPILE_ERROR\x10\x06\x12\x14\n\x10STATUS_EXECUTING\x10\x07\x12\x13\n\x0fSTATUS_FINISHED\x10\x08\x12\x14\n\x10STATUS_RUN_ERROR\x10\t\x12\x10\n\x0cSTATUS_ERROR\x10\n\x12\x16\n\x12STATUS_RUN_TIMEOUT\x10\x0b\x12\x13\n\x0fSTATUS_CANCELED\x10\x0c*\xae\x01\n\x15PrecompiledObjectType\x12\'\n#PRECOMPILED_OBJECT_TYPE_UNSPECIFIED\x10\x00\x12#\n\x1fPRECOMPILED_OBJECT_TYPE_EXAMPLE\x10\x01\x12 \n\x1cPRECOMPILED_OBJECT_TYPE_KATA\x10\x02\x12%\n!PRECOMPILED_OBJECT_TYPE_UNIT_TEST\x10\x03\x32\xec\x05\n\x11PlaygroundService\x12:\n\x07RunCode\x12\x16.api.v1.RunCodeRequest\x1a\x17.api.v1.RunCodeResponse\x12\x46\n\x0b\x43heckStatus\x12\x1a.api.v1.CheckStatusRequest\x1a\x1b.api.v1.CheckStatusResponse\x12I\n\x0cGetRunOutput\x12\x1b.api.v1.GetRunOutputRequest\x1a\x1c.api.v1.GetRunOutputResponse\x12\x46\n\x0bGetRunError\x12\x1a.api.v1.GetRunErrorRequest\x1a\x1b.api.v1.GetRunErrorResponse\x12U\n\x10GetCompileOutput\x12\x1f.api.v1.GetCompileOutputRequest\x1a .api.v1.GetCompileOutputResponse\x12\x37\n\x06\x43\x61ncel\x12\x15.api.v1.CancelRequest\x1a\x16.api.v1.CancelResponse\x12\x64\n\x15GetPrecompiledObjects\x12$.api.v1.GetPrecompiledObjectsRequest\x1a%.api.v1.GetPrecompiledObjectsResponse\x12i\n\x18GetPrecompiledObjectCode\x12#.api.v1.GetPrecompiledObjectRequest\x1a(.api.v1.GetPrecompiledObjectCodeResponse\x12_\n\x1aGetPrecompiledObjectOutput\x12#.api.v1.GetPrecompiledObjectRequest\x1a\x1c.api.v1.GetRunOutputResponseB8Z6beam.apache.org/playground/backend/internal;playgroundb\x06proto3' ) _SDK = _descriptor.EnumDescriptor( @@ -71,8 +71,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=1014, - serialized_end=1096, + serialized_start=1143, + serialized_end=1225, ) _sym_db.RegisterEnumDescriptor(_SDK) @@ -144,46 +144,56 @@ serialized_options=None, type=None, create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='STATUS_CANCELED', index=12, number=12, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), ], containing_type=None, serialized_options=None, - serialized_start=1099, - serialized_end=1390, + serialized_start=1228, + serialized_end=1540, ) _sym_db.RegisterEnumDescriptor(_STATUS) Status = enum_type_wrapper.EnumTypeWrapper(_STATUS) -_EXAMPLETYPE = _descriptor.EnumDescriptor( - name='ExampleType', - full_name='api.v1.ExampleType', +_PRECOMPILEDOBJECTTYPE = _descriptor.EnumDescriptor( + name='PrecompiledObjectType', + full_name='api.v1.PrecompiledObjectType', filename=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key, values=[ _descriptor.EnumValueDescriptor( - name='EXAMPLE_TYPE_DEFAULT', index=0, number=0, + name='PRECOMPILED_OBJECT_TYPE_UNSPECIFIED', index=0, number=0, serialized_options=None, type=None, create_key=_descriptor._internal_create_key), _descriptor.EnumValueDescriptor( - name='EXAMPLE_TYPE_KATA', index=1, number=1, + name='PRECOMPILED_OBJECT_TYPE_EXAMPLE', index=1, number=1, serialized_options=None, type=None, create_key=_descriptor._internal_create_key), _descriptor.EnumValueDescriptor( - name='EXAMPLE_TYPE_UNIT_TEST', index=2, number=2, + name='PRECOMPILED_OBJECT_TYPE_KATA', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='PRECOMPILED_OBJECT_TYPE_UNIT_TEST', index=3, number=3, serialized_options=None, type=None, create_key=_descriptor._internal_create_key), ], containing_type=None, serialized_options=None, - serialized_start=1392, - serialized_end=1482, + serialized_start=1543, + serialized_end=1717, ) -_sym_db.RegisterEnumDescriptor(_EXAMPLETYPE) +_sym_db.RegisterEnumDescriptor(_PRECOMPILEDOBJECTTYPE) -ExampleType = enum_type_wrapper.EnumTypeWrapper(_EXAMPLETYPE) +PrecompiledObjectType = enum_type_wrapper.EnumTypeWrapper(_PRECOMPILEDOBJECTTYPE) SDK_UNSPECIFIED = 0 SDK_JAVA = 1 SDK_GO = 2 @@ -201,9 +211,11 @@ STATUS_RUN_ERROR = 9 STATUS_ERROR = 10 STATUS_RUN_TIMEOUT = 11 -EXAMPLE_TYPE_DEFAULT = 0 -EXAMPLE_TYPE_KATA = 1 -EXAMPLE_TYPE_UNIT_TEST = 2 +STATUS_CANCELED = 12 +PRECOMPILED_OBJECT_TYPE_UNSPECIFIED = 0 +PRECOMPILED_OBJECT_TYPE_EXAMPLE = 1 +PRECOMPILED_OBJECT_TYPE_KATA = 2 +PRECOMPILED_OBJECT_TYPE_UNIT_TEST = 3 _RUNCODEREQUEST = _descriptor.Descriptor( name='RunCodeRequest', @@ -529,23 +541,78 @@ serialized_end=527, ) -_GETLISTOFEXAMPLESREQUEST = _descriptor.Descriptor( - name='GetListOfExamplesRequest', - full_name='api.v1.GetListOfExamplesRequest', +_CANCELREQUEST = _descriptor.Descriptor( + name='CancelRequest', + full_name='api.v1.CancelRequest', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='sdk', full_name='api.v1.GetListOfExamplesRequest.sdk', index=0, + name='pipeline_uuid', full_name='api.v1.CancelRequest.pipeline_uuid', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=529, + serialized_end=567, +) + +_CANCELRESPONSE = _descriptor.Descriptor( + name='CancelResponse', + full_name='api.v1.CancelResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=569, + serialized_end=585, +) + +_GETPRECOMPILEDOBJECTSREQUEST = _descriptor.Descriptor( + name='GetPrecompiledObjectsRequest', + full_name='api.v1.GetPrecompiledObjectsRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='sdk', full_name='api.v1.GetPrecompiledObjectsRequest.sdk', index=0, number=1, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='category', full_name='api.v1.GetListOfExamplesRequest.category', index=1, + name='category', full_name='api.v1.GetPrecompiledObjectsRequest.category', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -563,41 +630,41 @@ extension_ranges=[], oneofs=[ ], - serialized_start=529, - serialized_end=599, + serialized_start=587, + serialized_end=661, ) -_EXAMPLE = _descriptor.Descriptor( - name='Example', - full_name='api.v1.Example', +_PRECOMPILEDOBJECT = _descriptor.Descriptor( + name='PrecompiledObject', + full_name='api.v1.PrecompiledObject', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='example_uuid', full_name='api.v1.Example.example_uuid', index=0, + name='cloud_path', full_name='api.v1.PrecompiledObject.cloud_path', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='name', full_name='api.v1.Example.name', index=1, + name='name', full_name='api.v1.PrecompiledObject.name', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='description', full_name='api.v1.Example.description', index=2, + name='description', full_name='api.v1.PrecompiledObject.description', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='type', full_name='api.v1.Example.type', index=3, + name='type', full_name='api.v1.PrecompiledObject.type', index=3, number=4, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, @@ -615,8 +682,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=601, - serialized_end=702, + serialized_start=663, + serialized_end=782, ) _CATEGORIES_CATEGORY = _descriptor.Descriptor( @@ -635,7 +702,7 @@ is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='examples', full_name='api.v1.Categories.Category.examples', index=1, + name='precompiled_objects', full_name='api.v1.Categories.Category.precompiled_objects', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -653,8 +720,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=794, - serialized_end=862, + serialized_start=874, + serialized_end=963, ) _CATEGORIES = _descriptor.Descriptor( @@ -691,20 +758,20 @@ extension_ranges=[], oneofs=[ ], - serialized_start=705, - serialized_end=862, + serialized_start=785, + serialized_end=963, ) -_GETLISTOFEXAMPLESRESPONSE = _descriptor.Descriptor( - name='GetListOfExamplesResponse', - full_name='api.v1.GetListOfExamplesResponse', +_GETPRECOMPILEDOBJECTSRESPONSE = _descriptor.Descriptor( + name='GetPrecompiledObjectsResponse', + full_name='api.v1.GetPrecompiledObjectsResponse', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='sdk_examples', full_name='api.v1.GetListOfExamplesResponse.sdk_examples', index=0, + name='sdk_categories', full_name='api.v1.GetPrecompiledObjectsResponse.sdk_categories', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -722,20 +789,20 @@ extension_ranges=[], oneofs=[ ], - serialized_start=864, - serialized_end=933, + serialized_start=965, + serialized_end=1040, ) -_GETEXAMPLEREQUEST = _descriptor.Descriptor( - name='GetExampleRequest', - full_name='api.v1.GetExampleRequest', +_GETPRECOMPILEDOBJECTREQUEST = _descriptor.Descriptor( + name='GetPrecompiledObjectRequest', + full_name='api.v1.GetPrecompiledObjectRequest', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='example_uuid', full_name='api.v1.GetExampleRequest.example_uuid', index=0, + name='cloud_path', full_name='api.v1.GetPrecompiledObjectRequest.cloud_path', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -753,20 +820,20 @@ extension_ranges=[], oneofs=[ ], - serialized_start=935, - serialized_end=976, + serialized_start=1042, + serialized_end=1091, ) -_GETEXAMPLERESPONSE = _descriptor.Descriptor( - name='GetExampleResponse', - full_name='api.v1.GetExampleResponse', +_GETPRECOMPILEDOBJECTCODERESPONSE = _descriptor.Descriptor( + name='GetPrecompiledObjectCodeResponse', + full_name='api.v1.GetPrecompiledObjectCodeResponse', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='code', full_name='api.v1.GetExampleResponse.code', index=0, + name='code', full_name='api.v1.GetPrecompiledObjectCodeResponse.code', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -784,20 +851,20 @@ extension_ranges=[], oneofs=[ ], - serialized_start=978, - serialized_end=1012, + serialized_start=1093, + serialized_end=1141, ) _RUNCODEREQUEST.fields_by_name['sdk'].enum_type = _SDK _CHECKSTATUSRESPONSE.fields_by_name['status'].enum_type = _STATUS _GETCOMPILEOUTPUTRESPONSE.fields_by_name['compilation_status'].enum_type = _STATUS -_GETLISTOFEXAMPLESREQUEST.fields_by_name['sdk'].enum_type = _SDK -_EXAMPLE.fields_by_name['type'].enum_type = _EXAMPLETYPE -_CATEGORIES_CATEGORY.fields_by_name['examples'].message_type = _EXAMPLE +_GETPRECOMPILEDOBJECTSREQUEST.fields_by_name['sdk'].enum_type = _SDK +_PRECOMPILEDOBJECT.fields_by_name['type'].enum_type = _PRECOMPILEDOBJECTTYPE +_CATEGORIES_CATEGORY.fields_by_name['precompiled_objects'].message_type = _PRECOMPILEDOBJECT _CATEGORIES_CATEGORY.containing_type = _CATEGORIES _CATEGORIES.fields_by_name['sdk'].enum_type = _SDK _CATEGORIES.fields_by_name['categories'].message_type = _CATEGORIES_CATEGORY -_GETLISTOFEXAMPLESRESPONSE.fields_by_name['sdk_examples'].message_type = _CATEGORIES +_GETPRECOMPILEDOBJECTSRESPONSE.fields_by_name['sdk_categories'].message_type = _CATEGORIES DESCRIPTOR.message_types_by_name['RunCodeRequest'] = _RUNCODEREQUEST DESCRIPTOR.message_types_by_name['RunCodeResponse'] = _RUNCODERESPONSE DESCRIPTOR.message_types_by_name['CheckStatusRequest'] = _CHECKSTATUSREQUEST @@ -808,15 +875,17 @@ DESCRIPTOR.message_types_by_name['GetRunOutputResponse'] = _GETRUNOUTPUTRESPONSE DESCRIPTOR.message_types_by_name['GetRunErrorRequest'] = _GETRUNERRORREQUEST DESCRIPTOR.message_types_by_name['GetRunErrorResponse'] = _GETRUNERRORRESPONSE -DESCRIPTOR.message_types_by_name['GetListOfExamplesRequest'] = _GETLISTOFEXAMPLESREQUEST -DESCRIPTOR.message_types_by_name['Example'] = _EXAMPLE +DESCRIPTOR.message_types_by_name['CancelRequest'] = _CANCELREQUEST +DESCRIPTOR.message_types_by_name['CancelResponse'] = _CANCELRESPONSE +DESCRIPTOR.message_types_by_name['GetPrecompiledObjectsRequest'] = _GETPRECOMPILEDOBJECTSREQUEST +DESCRIPTOR.message_types_by_name['PrecompiledObject'] = _PRECOMPILEDOBJECT DESCRIPTOR.message_types_by_name['Categories'] = _CATEGORIES -DESCRIPTOR.message_types_by_name['GetListOfExamplesResponse'] = _GETLISTOFEXAMPLESRESPONSE -DESCRIPTOR.message_types_by_name['GetExampleRequest'] = _GETEXAMPLEREQUEST -DESCRIPTOR.message_types_by_name['GetExampleResponse'] = _GETEXAMPLERESPONSE +DESCRIPTOR.message_types_by_name['GetPrecompiledObjectsResponse'] = _GETPRECOMPILEDOBJECTSRESPONSE +DESCRIPTOR.message_types_by_name['GetPrecompiledObjectRequest'] = _GETPRECOMPILEDOBJECTREQUEST +DESCRIPTOR.message_types_by_name['GetPrecompiledObjectCodeResponse'] = _GETPRECOMPILEDOBJECTCODERESPONSE DESCRIPTOR.enum_types_by_name['Sdk'] = _SDK DESCRIPTOR.enum_types_by_name['Status'] = _STATUS -DESCRIPTOR.enum_types_by_name['ExampleType'] = _EXAMPLETYPE +DESCRIPTOR.enum_types_by_name['PrecompiledObjectType'] = _PRECOMPILEDOBJECTTYPE _sym_db.RegisterFileDescriptor(DESCRIPTOR) RunCodeRequest = _reflection.GeneratedProtocolMessageType('RunCodeRequest', (_message.Message,), { @@ -889,19 +958,34 @@ }) _sym_db.RegisterMessage(GetRunErrorResponse) -GetListOfExamplesRequest = _reflection.GeneratedProtocolMessageType('GetListOfExamplesRequest', (_message.Message,), { - 'DESCRIPTOR': _GETLISTOFEXAMPLESREQUEST, +CancelRequest = _reflection.GeneratedProtocolMessageType('CancelRequest', (_message.Message,), { + 'DESCRIPTOR': _CANCELREQUEST, '__module__': 'api_pb2' - # @@protoc_insertion_point(class_scope:api.v1.GetListOfExamplesRequest) + # @@protoc_insertion_point(class_scope:api.v1.CancelRequest) }) -_sym_db.RegisterMessage(GetListOfExamplesRequest) +_sym_db.RegisterMessage(CancelRequest) -Example = _reflection.GeneratedProtocolMessageType('Example', (_message.Message,), { - 'DESCRIPTOR': _EXAMPLE, +CancelResponse = _reflection.GeneratedProtocolMessageType('CancelResponse', (_message.Message,), { + 'DESCRIPTOR': _CANCELRESPONSE, '__module__': 'api_pb2' - # @@protoc_insertion_point(class_scope:api.v1.Example) + # @@protoc_insertion_point(class_scope:api.v1.CancelResponse) }) -_sym_db.RegisterMessage(Example) +_sym_db.RegisterMessage(CancelResponse) + +GetPrecompiledObjectsRequest = _reflection.GeneratedProtocolMessageType('GetPrecompiledObjectsRequest', + (_message.Message,), { + 'DESCRIPTOR': _GETPRECOMPILEDOBJECTSREQUEST, + '__module__': 'api_pb2' + # @@protoc_insertion_point(class_scope:api.v1.GetPrecompiledObjectsRequest) + }) +_sym_db.RegisterMessage(GetPrecompiledObjectsRequest) + +PrecompiledObject = _reflection.GeneratedProtocolMessageType('PrecompiledObject', (_message.Message,), { + 'DESCRIPTOR': _PRECOMPILEDOBJECT, + '__module__': 'api_pb2' + # @@protoc_insertion_point(class_scope:api.v1.PrecompiledObject) +}) +_sym_db.RegisterMessage(PrecompiledObject) Categories = _reflection.GeneratedProtocolMessageType('Categories', (_message.Message,), { @@ -918,26 +1002,29 @@ _sym_db.RegisterMessage(Categories) _sym_db.RegisterMessage(Categories.Category) -GetListOfExamplesResponse = _reflection.GeneratedProtocolMessageType('GetListOfExamplesResponse', (_message.Message,), { - 'DESCRIPTOR': _GETLISTOFEXAMPLESRESPONSE, - '__module__': 'api_pb2' - # @@protoc_insertion_point(class_scope:api.v1.GetListOfExamplesResponse) -}) -_sym_db.RegisterMessage(GetListOfExamplesResponse) +GetPrecompiledObjectsResponse = _reflection.GeneratedProtocolMessageType('GetPrecompiledObjectsResponse', + (_message.Message,), { + 'DESCRIPTOR': _GETPRECOMPILEDOBJECTSRESPONSE, + '__module__': 'api_pb2' + # @@protoc_insertion_point(class_scope:api.v1.GetPrecompiledObjectsResponse) + }) +_sym_db.RegisterMessage(GetPrecompiledObjectsResponse) -GetExampleRequest = _reflection.GeneratedProtocolMessageType('GetExampleRequest', (_message.Message,), { - 'DESCRIPTOR': _GETEXAMPLEREQUEST, - '__module__': 'api_pb2' - # @@protoc_insertion_point(class_scope:api.v1.GetExampleRequest) -}) -_sym_db.RegisterMessage(GetExampleRequest) +GetPrecompiledObjectRequest = _reflection.GeneratedProtocolMessageType('GetPrecompiledObjectRequest', + (_message.Message,), { + 'DESCRIPTOR': _GETPRECOMPILEDOBJECTREQUEST, + '__module__': 'api_pb2' + # @@protoc_insertion_point(class_scope:api.v1.GetPrecompiledObjectRequest) + }) +_sym_db.RegisterMessage(GetPrecompiledObjectRequest) -GetExampleResponse = _reflection.GeneratedProtocolMessageType('GetExampleResponse', (_message.Message,), { - 'DESCRIPTOR': _GETEXAMPLERESPONSE, - '__module__': 'api_pb2' - # @@protoc_insertion_point(class_scope:api.v1.GetExampleResponse) -}) -_sym_db.RegisterMessage(GetExampleResponse) +GetPrecompiledObjectCodeResponse = _reflection.GeneratedProtocolMessageType('GetPrecompiledObjectCodeResponse', + (_message.Message,), { + 'DESCRIPTOR': _GETPRECOMPILEDOBJECTCODERESPONSE, + '__module__': 'api_pb2' + # @@protoc_insertion_point(class_scope:api.v1.GetPrecompiledObjectCodeResponse) + }) +_sym_db.RegisterMessage(GetPrecompiledObjectCodeResponse) DESCRIPTOR._options = None @@ -948,8 +1035,8 @@ index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=1485, - serialized_end=2106, + serialized_start=1720, + serialized_end=2468, methods=[ _descriptor.MethodDescriptor( name='RunCode', @@ -1002,31 +1089,41 @@ create_key=_descriptor._internal_create_key, ), _descriptor.MethodDescriptor( - name='GetListOfExamples', - full_name='api.v1.PlaygroundService.GetListOfExamples', + name='Cancel', + full_name='api.v1.PlaygroundService.Cancel', index=5, containing_service=None, - input_type=_GETLISTOFEXAMPLESREQUEST, - output_type=_GETLISTOFEXAMPLESRESPONSE, + input_type=_CANCELREQUEST, + output_type=_CANCELRESPONSE, serialized_options=None, create_key=_descriptor._internal_create_key, ), _descriptor.MethodDescriptor( - name='GetExample', - full_name='api.v1.PlaygroundService.GetExample', + name='GetPrecompiledObjects', + full_name='api.v1.PlaygroundService.GetPrecompiledObjects', index=6, containing_service=None, - input_type=_GETEXAMPLEREQUEST, - output_type=_GETEXAMPLERESPONSE, + input_type=_GETPRECOMPILEDOBJECTSREQUEST, + output_type=_GETPRECOMPILEDOBJECTSRESPONSE, serialized_options=None, create_key=_descriptor._internal_create_key, ), _descriptor.MethodDescriptor( - name='GetExampleOutput', - full_name='api.v1.PlaygroundService.GetExampleOutput', + name='GetPrecompiledObjectCode', + full_name='api.v1.PlaygroundService.GetPrecompiledObjectCode', index=7, containing_service=None, - input_type=_GETEXAMPLEREQUEST, + input_type=_GETPRECOMPILEDOBJECTREQUEST, + output_type=_GETPRECOMPILEDOBJECTCODERESPONSE, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='GetPrecompiledObjectOutput', + full_name='api.v1.PlaygroundService.GetPrecompiledObjectOutput', + index=8, + containing_service=None, + input_type=_GETPRECOMPILEDOBJECTREQUEST, output_type=_GETRUNOUTPUTRESPONSE, serialized_options=None, create_key=_descriptor._internal_create_key, diff --git a/playground/infrastructure/api/v1/api_pb2_grpc.py b/playground/infrastructure/api/v1/api_pb2_grpc.py index ef83f6d2f858..fb19baf8db12 100644 --- a/playground/infrastructure/api/v1/api_pb2_grpc.py +++ b/playground/infrastructure/api/v1/api_pb2_grpc.py @@ -53,19 +53,24 @@ def __init__(self, channel): request_serializer=api__pb2.GetCompileOutputRequest.SerializeToString, response_deserializer=api__pb2.GetCompileOutputResponse.FromString, ) - self.GetListOfExamples = channel.unary_unary( - '/api.v1.PlaygroundService/GetListOfExamples', - request_serializer=api__pb2.GetListOfExamplesRequest.SerializeToString, - response_deserializer=api__pb2.GetListOfExamplesResponse.FromString, + self.Cancel = channel.unary_unary( + '/api.v1.PlaygroundService/Cancel', + request_serializer=api__pb2.CancelRequest.SerializeToString, + response_deserializer=api__pb2.CancelResponse.FromString, ) - self.GetExample = channel.unary_unary( - '/api.v1.PlaygroundService/GetExample', - request_serializer=api__pb2.GetExampleRequest.SerializeToString, - response_deserializer=api__pb2.GetExampleResponse.FromString, + self.GetPrecompiledObjects = channel.unary_unary( + '/api.v1.PlaygroundService/GetPrecompiledObjects', + request_serializer=api__pb2.GetPrecompiledObjectsRequest.SerializeToString, + response_deserializer=api__pb2.GetPrecompiledObjectsResponse.FromString, ) - self.GetExampleOutput = channel.unary_unary( - '/api.v1.PlaygroundService/GetExampleOutput', - request_serializer=api__pb2.GetExampleRequest.SerializeToString, + self.GetPrecompiledObjectCode = channel.unary_unary( + '/api.v1.PlaygroundService/GetPrecompiledObjectCode', + request_serializer=api__pb2.GetPrecompiledObjectRequest.SerializeToString, + response_deserializer=api__pb2.GetPrecompiledObjectCodeResponse.FromString, + ) + self.GetPrecompiledObjectOutput = channel.unary_unary( + '/api.v1.PlaygroundService/GetPrecompiledObjectOutput', + request_serializer=api__pb2.GetPrecompiledObjectRequest.SerializeToString, response_deserializer=api__pb2.GetRunOutputResponse.FromString, ) @@ -108,22 +113,29 @@ def GetCompileOutput(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def GetListOfExamples(self, request, context): - """Get the list of precompiled examples. + def Cancel(self, request, context): + """Cancel code processing + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetPrecompiledObjects(self, request, context): + """Get all precompiled objects from the cloud storage. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def GetExample(self, request, context): - """Get the code of an example. + def GetPrecompiledObjectCode(self, request, context): + """Get the code of an PrecompiledObject. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def GetExampleOutput(self, request, context): - """Get the precompiled details of an example. + def GetPrecompiledObjectOutput(self, request, context): + """Get the precompiled details of an PrecompiledObject. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -157,19 +169,24 @@ def add_PlaygroundServiceServicer_to_server(servicer, server): request_deserializer=api__pb2.GetCompileOutputRequest.FromString, response_serializer=api__pb2.GetCompileOutputResponse.SerializeToString, ), - 'GetListOfExamples': grpc.unary_unary_rpc_method_handler( - servicer.GetListOfExamples, - request_deserializer=api__pb2.GetListOfExamplesRequest.FromString, - response_serializer=api__pb2.GetListOfExamplesResponse.SerializeToString, + 'Cancel': grpc.unary_unary_rpc_method_handler( + servicer.Cancel, + request_deserializer=api__pb2.CancelRequest.FromString, + response_serializer=api__pb2.CancelResponse.SerializeToString, + ), + 'GetPrecompiledObjects': grpc.unary_unary_rpc_method_handler( + servicer.GetPrecompiledObjects, + request_deserializer=api__pb2.GetPrecompiledObjectsRequest.FromString, + response_serializer=api__pb2.GetPrecompiledObjectsResponse.SerializeToString, ), - 'GetExample': grpc.unary_unary_rpc_method_handler( - servicer.GetExample, - request_deserializer=api__pb2.GetExampleRequest.FromString, - response_serializer=api__pb2.GetExampleResponse.SerializeToString, + 'GetPrecompiledObjectCode': grpc.unary_unary_rpc_method_handler( + servicer.GetPrecompiledObjectCode, + request_deserializer=api__pb2.GetPrecompiledObjectRequest.FromString, + response_serializer=api__pb2.GetPrecompiledObjectCodeResponse.SerializeToString, ), - 'GetExampleOutput': grpc.unary_unary_rpc_method_handler( - servicer.GetExampleOutput, - request_deserializer=api__pb2.GetExampleRequest.FromString, + 'GetPrecompiledObjectOutput': grpc.unary_unary_rpc_method_handler( + servicer.GetPrecompiledObjectOutput, + request_deserializer=api__pb2.GetPrecompiledObjectRequest.FromString, response_serializer=api__pb2.GetRunOutputResponse.SerializeToString, ), } @@ -268,52 +285,69 @@ def GetCompileOutput(request, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def GetListOfExamples(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetListOfExamples', - api__pb2.GetListOfExamplesRequest.SerializeToString, - api__pb2.GetListOfExamplesResponse.FromString, + def Cancel(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/Cancel', + api__pb2.CancelRequest.SerializeToString, + api__pb2.CancelResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def GetExample(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetExample', - api__pb2.GetExampleRequest.SerializeToString, - api__pb2.GetExampleResponse.FromString, + def GetPrecompiledObjects(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetPrecompiledObjects', + api__pb2.GetPrecompiledObjectsRequest.SerializeToString, + api__pb2.GetPrecompiledObjectsResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def GetExampleOutput(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetExampleOutput', - api__pb2.GetExampleRequest.SerializeToString, + def GetPrecompiledObjectCode(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetPrecompiledObjectCode', + api__pb2.GetPrecompiledObjectRequest.SerializeToString, + api__pb2.GetPrecompiledObjectCodeResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetPrecompiledObjectOutput(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetPrecompiledObjectOutput', + api__pb2.GetPrecompiledObjectRequest.SerializeToString, api__pb2.GetRunOutputResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/playground/infrastructure/ci_cd.py b/playground/infrastructure/ci_cd.py index 1400cc951345..a6947231e9a8 100644 --- a/playground/infrastructure/ci_cd.py +++ b/playground/infrastructure/ci_cd.py @@ -12,12 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import asyncio import os from cd_helper import CDHelper from ci_helper import CIHelper -from helper import find_examples +from helper import find_examples, get_supported_categories from logger import setup_logger @@ -27,8 +28,10 @@ def ci_step(): """ setup_logger() root_dir = os.getenv("BEAM_ROOT_DIR") + categories_file = os.getenv("BEAM_EXAMPLE_CATEGORIES") + supported_categories = get_supported_categories(categories_file) ci_helper = CIHelper() - examples = find_examples(root_dir) + examples = find_examples(root_dir, supported_categories) asyncio.run(ci_helper.verify_examples(examples)) @@ -38,6 +41,8 @@ def cd_step(): """ setup_logger() root_dir = os.getenv("BEAM_ROOT_DIR") + categories_file = os.getenv("BEAM_EXAMPLE_CATEGORIES") + supported_categories = get_supported_categories(categories_file) cd_helper = CDHelper() - examples = find_examples(root_dir) + examples = find_examples(root_dir, supported_categories) cd_helper.store_examples(examples) diff --git a/playground/infrastructure/ci_helper.py b/playground/infrastructure/ci_helper.py index 773040ae35ee..2744c4d27ba7 100644 --- a/playground/infrastructure/ci_helper.py +++ b/playground/infrastructure/ci_helper.py @@ -38,7 +38,7 @@ async def verify_examples(self, examples: List[Example]): 2. Group code of examples by their SDK. 3. Run processing for all examples to verify examples' code. """ - get_statuses(examples) + await get_statuses(examples) await self._verify_examples_status(examples) async def _verify_examples_status(self, examples: List[Example]): diff --git a/playground/infrastructure/config.py b/playground/infrastructure/config.py index 79ad1ceaad80..1c006be5d99e 100644 --- a/playground/infrastructure/config.py +++ b/playground/infrastructure/config.py @@ -14,11 +14,10 @@ # limitations under the License. import os + from dataclasses import dataclass from api.v1.api_pb2 import STATUS_VALIDATION_ERROR, STATUS_ERROR, STATUS_PREPARATION_ERROR, STATUS_COMPILE_ERROR, \ - STATUS_RUN_TIMEOUT, STATUS_RUN_ERROR - -from api.v1.api_pb2 import SDK_JAVA + STATUS_RUN_TIMEOUT, STATUS_RUN_ERROR, SDK_JAVA, SDK_GO, SDK_PYTHON @dataclass(frozen=True) @@ -26,4 +25,15 @@ class Config: SERVER_ADDRESS = os.getenv("SERVER_ADDRESS", "localhost:8080") ERROR_STATUSES = [STATUS_VALIDATION_ERROR, STATUS_ERROR, STATUS_PREPARATION_ERROR, STATUS_COMPILE_ERROR, STATUS_RUN_TIMEOUT, STATUS_RUN_ERROR] - SUPPORTED_SDK = {'java': SDK_JAVA} + SUPPORTED_SDK = {'java': SDK_JAVA, 'go': SDK_GO, 'py': SDK_PYTHON} + BEAM_PLAYGROUND_TITLE = "Beam-playground:\n" + BEAM_PLAYGROUND = "Beam-playground" + PAUSE_DELAY = 10 + + +@dataclass(frozen=True) +class TagFields: + NAME: str = "name" + DESCRIPTION: str = "description" + MULTIFILE: str = "multifile" + CATEGORIES: str = "categories" diff --git a/playground/infrastructure/grpc_client.py b/playground/infrastructure/grpc_client.py index 3b5279952c97..f908dc5f3a1d 100644 --- a/playground/infrastructure/grpc_client.py +++ b/playground/infrastructure/grpc_client.py @@ -12,8 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import uuid +import uuid import grpc from api.v1 import api_pb2_grpc, api_pb2 diff --git a/playground/infrastructure/helper.py b/playground/infrastructure/helper.py index 0c1a72c88a3f..0649fd466aa8 100644 --- a/playground/infrastructure/helper.py +++ b/playground/infrastructure/helper.py @@ -13,9 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass +import asyncio +import logging +import os +import yaml + +from dataclasses import dataclass, fields from typing import List -from api.v1.api_pb2 import SDK_UNSPECIFIED, STATUS_UNSPECIFIED +from yaml import YAMLError +from config import Config, TagFields +from collections import namedtuple +from api.v1.api_pb2 import Sdk, SDK_UNSPECIFIED, STATUS_UNSPECIFIED, STATUS_VALIDATING, \ + STATUS_PREPARING, STATUS_COMPILING, STATUS_EXECUTING +from grpc_client import GRPCClient + +Tag = namedtuple("Tag", [TagFields.NAME, TagFields.DESCRIPTION, TagFields.MULTIFILE, TagFields.CATEGORIES]) @dataclass @@ -30,9 +42,10 @@ class Example: code: str output: str status: STATUS_UNSPECIFIED + tag: Tag -def find_examples(work_dir: str) -> List[Example]: +def find_examples(work_dir: str, supported_categories: List[str]) -> List[Example]: """ Find and return beam examples. @@ -44,29 +57,240 @@ def find_examples(work_dir: str) -> List[Example]: categories: - category-1 - category-2 + If some example contain beam tag with incorrect format raise an error. Args: work_dir: directory where to search examples. + supported_categories: list of supported categories. Returns: List of Examples. """ - # TODO [BEAM-13135] Implement - pass + has_error = False + examples = [] + for root, _, files in os.walk(work_dir): + for filename in files: + filepath = os.path.join(root, filename) + error_during_check_file = _check_file(examples, filename, filepath, supported_categories) + has_error = has_error or error_during_check_file + if has_error: + raise ValueError("Some of the beam examples contain beam playground tag with an incorrect format") + return examples + + +async def get_statuses(examples: List[Example]): + """ + Receive status and update example.status and example.pipeline_id for each example + + Args: + examples: beam examples for processing and updating statuses and pipeline_id values. + """ + tasks = [] + client = GRPCClient() + for example in examples: + tasks.append(_update_example_status(example, client)) + await asyncio.gather(*tasks) + + +def get_tag(filepath): + """ + Parse file by filepath and find beam tag + + Args: + filepath: path of the file + + Returns: + If file contains tag, returns tag as a map. + If file doesn't contain tag, returns None + """ + add_to_yaml = False + yaml_string = "" + + with open(filepath) as parsed_file: + lines = parsed_file.readlines() + + for line in lines: + line = line.replace("//", "").replace("#", "") + if add_to_yaml is False: + if line.lstrip() == Config.BEAM_PLAYGROUND_TITLE: + add_to_yaml = True + yaml_string += line.lstrip() + else: + yaml_with_new_string = yaml_string + line + try: + yaml.load(yaml_with_new_string, Loader=yaml.SafeLoader) + yaml_string += line + except YAMLError: + break + + if add_to_yaml: + tag_object = yaml.load(yaml_string, Loader=yaml.SafeLoader) + return tag_object[Config.BEAM_PLAYGROUND] + + return None + + +def _check_file(examples, filename, filepath, supported_categories): + """ + Check file by filepath for matching to beam example. If file is beam example, then add it to list of examples + + Args: + examples: list of examples. + filename: name of the file. + filepath: path to the file. + supported_categories: list of supported categories. + + Returns: + True if file has beam playground tag with incorrect format. + False if file has correct beam playground tag. + False if file doesn't contains beam playground tag. + """ + if filepath.endswith("infrastructure/helper.py"): + return False + + has_error = False + extension = filepath.split(os.extsep)[-1] + if extension in Config.SUPPORTED_SDK: + tag = get_tag(filepath) + if tag: + if _validate(tag, supported_categories) is False: + logging.error(filepath + "contains beam playground tag with incorrect format") + has_error = True + else: + examples.append(_get_example(filepath, filename, tag)) + return has_error + + +def get_supported_categories(categories_path: str) -> List[str]: + """ + Return list of supported categories from categories_path file + + Args: + categories_path: path to the file with categories. + + Returns: + All supported categories as a list. + """ + with open(categories_path) as supported_categories: + yaml_object = yaml.load(supported_categories.read(), Loader=yaml.SafeLoader) + return yaml_object[TagFields.CATEGORIES] + + +def _get_example(filepath: str, filename: str, tag: dict) -> Example: + """ + Return an Example by filepath and filename. + + Args: + tag: tag of the example. + filepath: path of the example's file. + filename: name of the example's file. + + Returns: + Parsed Example object. + """ + name = _get_name(filename) + sdk = _get_sdk(filename) + with open(filepath) as parsed_file: + content = parsed_file.read() + + return Example(name, "", sdk, filepath, content, "", STATUS_UNSPECIFIED, Tag(**tag)) + + +def _validate(tag: dict, supported_categories: List[str]) -> bool: + """ + Validate all tag's fields + + Validate that tag contains all required fields and all fields have required format. + + Args: + tag: beam tag to validate. + supported_categories: list of supported categories. + + Returns: + In case tag is valid, True + In case tag is not valid, False + """ + valid = True + for field in fields(TagFields): + if tag.get(field.default) is None: + logging.error("tag doesn't contain " + field.default + " field: " + tag.__str__() + "\n" + + "Please, double-check that this field exists in the beam playground tag. " + + "If you are sure that this field exists in the tag double-check the format of indenting.") + valid = False + + multifile = tag.get(TagFields.MULTIFILE) + if (multifile is not None) and (str(multifile).lower() not in ["true", "false"]): + logging.error("tag's field multifile is incorrect: " + tag.__str__() + "\n" + + "multifile variable should be boolean format, but tag contains: " + str(multifile)) + valid = False + + categories = tag.get(TagFields.CATEGORIES) + if categories is not None: + if type(categories) is not list: + logging.error("tag's field categories is incorrect: " + tag.__str__() + "\n" + + "categories variable should be list format, but tag contains: " + str(type(categories))) + valid = False + else: + for category in categories: + if category not in supported_categories: + logging.error("tag contains unsupported category: " + category + "\n" + + "If you are sure that " + category + " category should be placed in " + + "Beam Playground, you can add it to the `playground/categories.yaml` file") + valid = False + return valid + + +def _get_name(filename: str) -> str: + """ + Return name of the example by his filepath. + + Get name of the example by his filename. + + Args: + filename: filename of the beam example file. + + Returns: + example's name. + """ + return filename.split(os.extsep)[0] + + +def _get_sdk(filename: str) -> Sdk: + """ + Return SDK of example by his filename. + + Get extension of the example's file and returns associated SDK. + + Args: + filename: filename of the beam example. + + Returns: + Sdk according to file extension. + """ + extension = filename.split(os.extsep)[-1] + if extension in Config.SUPPORTED_SDK: + return Config.SUPPORTED_SDK[extension] + else: + raise ValueError(extension + " is not supported") -def get_statuses(examples: List[Example]): +async def _update_example_status(example: Example, client: GRPCClient): """ - Receive statuses for examples and update example.status and example.pipelineId + Receive status for examples and update example.status and pipeline_id Use client to send requests to the backend: 1. Start code processing. 2. Ping the backend while status is STATUS_VALIDATING/STATUS_PREPARING/STATUS_COMPILING/STATUS_EXECUTING - Update example.pipelineId with resulting pipelineId. Update example.status with resulting status. Args: - examples: beam examples for processing and updating statuses. + example: beam example for processing and updating status and pipeline_id. + client: client to send requests to the server. """ - # TODO [BEAM-13267] Implement - pass + pipeline_id = await client.run_code(example.code, example.sdk) + example.pipeline_id = pipeline_id + status = await client.check_status(pipeline_id) + while status in [STATUS_VALIDATING, STATUS_PREPARING, STATUS_COMPILING, STATUS_EXECUTING]: + await asyncio.sleep(Config.PAUSE_DELAY) + status = await client.check_status(pipeline_id) + example.status = status diff --git a/playground/infrastructure/requirements.txt b/playground/infrastructure/requirements.txt new file mode 100644 index 000000000000..db6279e740ee --- /dev/null +++ b/playground/infrastructure/requirements.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +grpcio-tools==1.41.0 +grpcio==1.41.1 +mock==4.0.3 +protobuf==3.19.1 +pytest==6.2.5 +PyYAML==6.0 diff --git a/playground/infrastructure/test_ci_cd.py b/playground/infrastructure/test_ci_cd.py index 07032c70361b..a030602ced79 100644 --- a/playground/infrastructure/test_ci_cd.py +++ b/playground/infrastructure/test_ci_cd.py @@ -20,27 +20,33 @@ @mock.patch('ci_helper.CIHelper.verify_examples') @mock.patch('ci_cd.find_examples') +@mock.patch('ci_cd.get_supported_categories') @mock.patch('ci_cd.os.getenv') -def test_ci_step(mock_os_getenv, mock_find_examples, mock_verify_examples): +def test_ci_step(mock_os_getenv, mock_get_supported_categories, mock_find_examples, mock_verify_examples): mock_os_getenv.return_value = "MOCK_VALUE" + mock_get_supported_categories.return_value = [] mock_find_examples.return_value = [] ci_step() - mock_os_getenv.assert_called_once_with("BEAM_ROOT_DIR") - mock_find_examples.assert_called_once_with("MOCK_VALUE") + mock_os_getenv.assert_has_calls([mock.call("BEAM_ROOT_DIR"), mock.call("BEAM_EXAMPLE_CATEGORIES")]) + mock_get_supported_categories.assert_called_once_with("MOCK_VALUE") + mock_find_examples.assert_called_once_with("MOCK_VALUE", []) mock_verify_examples.assert_called_once_with([]) @mock.patch('cd_helper.CDHelper.store_examples') @mock.patch('ci_cd.find_examples') +@mock.patch('ci_cd.get_supported_categories') @mock.patch('ci_cd.os.getenv') -def test_cd_step(mock_os_getenv, mock_find_examples, mock_store_examples): +def test_cd_step(mock_os_getenv, mock_get_supported_categories, mock_find_examples, mock_store_examples): mock_os_getenv.return_value = "MOCK_VALUE" + mock_get_supported_categories.return_value = [] mock_find_examples.return_value = [] cd_step() - mock_os_getenv.assert_called_once_with("BEAM_ROOT_DIR") - mock_find_examples.assert_called_once_with("MOCK_VALUE") + mock_os_getenv.assert_has_calls([mock.call("BEAM_ROOT_DIR"), mock.call("BEAM_EXAMPLE_CATEGORIES")]) + mock_get_supported_categories.assert_called_once_with("MOCK_VALUE") + mock_find_examples.assert_called_once_with("MOCK_VALUE", []) mock_store_examples.assert_called_once_with([]) diff --git a/playground/infrastructure/test_ci_helper.py b/playground/infrastructure/test_ci_helper.py index 65cc8285159f..0ccbd8995742 100644 --- a/playground/infrastructure/test_ci_helper.py +++ b/playground/infrastructure/test_ci_helper.py @@ -14,15 +14,17 @@ # limitations under the License. import mock +import pytest from ci_helper import CIHelper +@pytest.mark.asyncio @mock.patch('ci_helper.CIHelper._verify_examples_status') @mock.patch('ci_helper.get_statuses') -def test_verify_examples(mock_get_statuses, mock_verify_examples_statuses): +async def test_verify_examples(mock_get_statuses, mock_verify_examples_statuses): helper = CIHelper() - helper.verify_examples([]) + await helper.verify_examples([]) mock_get_statuses.assert_called_once_with([]) mock_verify_examples_statuses.assert_called_once_with([]) diff --git a/playground/infrastructure/test_grpc_client.py b/playground/infrastructure/test_grpc_client.py index 8302fa2ba9c8..1e63e1f882a4 100644 --- a/playground/infrastructure/test_grpc_client.py +++ b/playground/infrastructure/test_grpc_client.py @@ -14,9 +14,9 @@ # limitations under the License. import uuid -from unittest.mock import AsyncMock import pytest +from unittest.mock import AsyncMock from api.v1 import api_pb2 from grpc_client import GRPCClient diff --git a/playground/infrastructure/test_helper.py b/playground/infrastructure/test_helper.py new file mode 100644 index 000000000000..740a602c7a7e --- /dev/null +++ b/playground/infrastructure/test_helper.py @@ -0,0 +1,220 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pytest + +from unittest.mock import mock_open +from api.v1.api_pb2 import SDK_UNSPECIFIED, STATUS_UNSPECIFIED, SDK_JAVA, SDK_PYTHON, SDK_GO, \ + STATUS_VALIDATING, STATUS_FINISHED +from grpc_client import GRPCClient +from helper import find_examples, Example, _get_example, _get_name, _get_sdk, get_tag, _validate, Tag, get_statuses, \ + _update_example_status, get_supported_categories, _check_file + + +@mock.patch('helper._check_file') +@mock.patch('helper.os.walk') +def test_find_examples_with_valid_tag(mock_os_walk, mock_check_file): + mock_os_walk.return_value = [("/root", (), ("file.java",))] + mock_check_file.return_value = False + + result = find_examples("", []) + + assert result == [] + mock_os_walk.assert_called_once_with("") + mock_check_file.assert_called_once_with([], "file.java", "/root/file.java", []) + + +@mock.patch('helper._check_file') +@mock.patch('helper.os.walk') +def test_find_examples_with_invalid_tag(mock_os_walk, mock_check_file): + mock_os_walk.return_value = [("/root", (), ("file.java",))] + mock_check_file.return_value = True + + with pytest.raises(ValueError, + match="Some of the beam examples contain beam playground tag with an incorrect format"): + find_examples("", []) + + mock_os_walk.assert_called_once_with("") + mock_check_file.assert_called_once_with([], "file.java", "/root/file.java", []) + + +@pytest.mark.asyncio +@mock.patch('helper.GRPCClient') +@mock.patch('helper._update_example_status') +async def test_get_statuses(mock_update_example_status, mock_grpc_client): + example = Example("file", "pipeline_id", SDK_UNSPECIFIED, "root/file.extension", "code", "output", + STATUS_UNSPECIFIED, {"name": "Name"}) + client = None + + mock_grpc_client.return_value = client + + await get_statuses([example]) + + mock_update_example_status.assert_called_once_with(example, client) + + +@mock.patch('builtins.open', mock_open(read_data="...\n# Beam-playground:\n# name: Name\n\nimport ...")) +def test_get_tag_when_tag_is_exists(): + result = get_tag("") + + assert result.get("name") == "Name" + + +@mock.patch('builtins.open', mock_open(read_data="...\n...")) +def test_get_tag_when_tag_does_not_exist(): + result = get_tag("") + + assert result is None + + +@mock.patch('helper._get_example') +@mock.patch('helper._validate') +@mock.patch('helper.get_tag') +def test__check_file_with_correct_tag(mock_get_tag, mock_validate, mock_get_example): + tag = {"name": "Name"} + example = Example("filename", "", SDK_UNSPECIFIED, "/root/filename.java", "data", "", STATUS_UNSPECIFIED, + Tag("Name", "Description", False, [])) + examples = [] + + mock_get_tag.return_value = tag + mock_validate.return_value = True + mock_get_example.return_value = example + + result = _check_file(examples, "filename.java", "/root/filename.java", []) + + assert result is False + assert len(examples) == 1 + assert examples[0] == example + mock_get_tag.assert_called_once_with("/root/filename.java") + mock_validate.assert_called_once_with(tag, []) + mock_get_example.assert_called_once_with("/root/filename.java", "filename.java", tag) + + +@mock.patch('helper._validate') +@mock.patch('helper.get_tag') +def test__check_file_with_incorrect_tag(mock_get_tag, mock_validate): + tag = {"name": "Name"} + examples = [] + + mock_get_tag.return_value = tag + mock_validate.return_value = False + + result = _check_file(examples, "filename.java", "/root/filename.java", []) + + assert result is True + assert len(examples) == 0 + mock_get_tag.assert_called_once_with("/root/filename.java") + mock_validate.assert_called_once_with(tag, []) + + +@mock.patch('builtins.open', mock_open(read_data="categories:\n - category")) +def test_get_supported_categories(): + result = get_supported_categories("") + + assert len(result) == 1 + assert result[0] == "category" + + +@mock.patch('builtins.open', mock_open(read_data="data")) +@mock.patch('helper._get_sdk') +@mock.patch('helper._get_name') +def test__get_example(mock_get_name, mock_get_sdk): + mock_get_name.return_value = "filepath" + mock_get_sdk.return_value = SDK_UNSPECIFIED + + result = _get_example("/root/filepath.extension", "filepath.extension", + {'name': 'Name', "description": "Description", "multifile": "False", "categories": [""]}) + + assert result == Example("filepath", "", SDK_UNSPECIFIED, "/root/filepath.extension", "data", "", + STATUS_UNSPECIFIED, + Tag("Name", "Description", "False", [""])) + mock_get_name.assert_called_once_with("filepath.extension") + mock_get_sdk.assert_called_once_with("filepath.extension") + + +def test__validate_without_name_field(): + tag = {} + assert _validate(tag, []) is False + + +def test__validate_without_description_field(): + tag = {"name": "Name"} + assert _validate(tag, []) is False + + +def test__validate_without_multifile_field(): + tag = {"name": "Name", "description": "Description"} + assert _validate(tag, []) is False + + +def test__validate_with_incorrect_multifile_field(): + tag = {"name": "Name", "description": "Description", "multifile": "Multifile"} + assert _validate(tag, []) is False + + +def test__validate_without_categories_field(): + tag = {"name": "Name", "description": "Description", "multifile": "true"} + assert _validate(tag, []) is False + + +def test__validate_without_incorrect_categories_field(): + tag = {"name": "Name", "description": "Description", "multifile": "true", "categories": "Categories"} + assert _validate(tag, []) is False + + +def test__validate_with_not_supported_category(): + tag = {"name": "Name", "description": "Description", "multifile": "true", "categories": ["category1"]} + assert _validate(tag, ["category"]) is False + + +def test__validate_with_all_fields(): + tag = {"name": "Name", "description": "Description", "multifile": "true", "categories": ["category"]} + assert _validate(tag, ["category"]) is True + + +def test__get_name(): + result = _get_name("filepath.extension") + + assert result == "filepath" + + +def test__get_sdk_with_supported_extension(): + assert _get_sdk("filename.java") == SDK_JAVA + assert _get_sdk("filename.go") == SDK_GO + assert _get_sdk("filename.py") == SDK_PYTHON + + +def test__get_sdk_with_unsupported_extension(): + with pytest.raises(ValueError, match="extension is not supported"): + _get_sdk("filename.extension") + + +@pytest.mark.asyncio +@mock.patch('grpc_client.GRPCClient.check_status') +@mock.patch('grpc_client.GRPCClient.run_code') +async def test__update_example_status(mock_grpc_client_run_code, mock_grpc_client_check_status): + example = Example("file", "pipeline_id", SDK_UNSPECIFIED, "root/file.extension", "code", "output", + STATUS_UNSPECIFIED, {"name": "Name"}) + + mock_grpc_client_run_code.return_value = "pipeline_id" + mock_grpc_client_check_status.side_effect = [STATUS_VALIDATING, STATUS_FINISHED] + + await _update_example_status(example, GRPCClient()) + + assert example.pipeline_id == "pipeline_id" + assert example.status == STATUS_FINISHED + mock_grpc_client_run_code.assert_called_once_with(example.code, example.sdk) + mock_grpc_client_check_status.assert_has_calls([mock.call("pipeline_id")])