From b73f5f70ac4184b56a0a03922731c5f2f69b9566 Mon Sep 17 00:00:00 2001 From: Sergey Kalinin <91209855+snkalinin@users.noreply.github.com> Date: Tue, 16 Nov 2021 14:59:11 +0300 Subject: [PATCH 1/4] [Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83) * Update workflows for playground * Attempt to fix tests * Remove continue on error to catch errors * Fix linter problem for backend dockerfile * Update folder to run backend go linter * Moved flutter test to execution via gradle tasks --- .../workflows/build_playground_backend.yml | 14 ++++++++----- .../workflows/build_playground_frontend.yml | 20 +++++++------------ playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index 96e8aae9b0d5..bd2868768246 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,10 +17,8 @@ name: Build And Deploy Playground Backend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -45,12 +43,14 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint - run: "golangci-lint run internal/..." +# run: "golangci-lint run internal/..." + run: "golangci-lint run cmd/server/..." working-directory: playground/backend/ - continue-on-error: true + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue + run: rm ~/.m2/settings.xml - name: Run Tests run: ./gradlew playground:backend:test - continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,12 +62,16 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index 9c2d56a52428..f93e330442b0 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,12 +17,10 @@ name: Build And Deploy Playground Frontend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/backend/**'] + paths: ['playground/frontend/**'] workflow_dispatch: jobs: @@ -43,18 +41,10 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' - - name: Prepare Flutter lint - run: "flutter pub add flutter_lints --dev" - working-directory: playground/frontend/ - continue-on-error: true - name: Run Lint - run: "flutter analyze" - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:analyze - name: Run Tests - run: flutter test - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:test - name: install npm uses: actions/setup-node@v2 with: @@ -66,12 +56,16 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index d7275020dcfa..8846368ff8b6 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE +ARG BASE_IMAGE=apache/beam_java8_sdk:latest FROM golang:1.17-buster AS build # Setup Go Environment From d449275b8ebd623760b34981f56c8f62b140ad9d Mon Sep 17 00:00:00 2001 From: Ilya Date: Tue, 16 Nov 2021 15:07:49 +0300 Subject: [PATCH 2/4] Revert "[Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83)" (#88) This reverts commit b73f5f70ac4184b56a0a03922731c5f2f69b9566. --- .../workflows/build_playground_backend.yml | 14 +++++-------- .../workflows/build_playground_frontend.yml | 20 ++++++++++++------- playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index bd2868768246..96e8aae9b0d5 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,8 +17,10 @@ name: Build And Deploy Playground Backend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -43,14 +45,12 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint -# run: "golangci-lint run internal/..." - run: "golangci-lint run cmd/server/..." + run: "golangci-lint run internal/..." working-directory: playground/backend/ - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: rm ~/.m2/settings.xml + continue-on-error: true - name: Run Tests run: ./gradlew playground:backend:test + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,16 +62,12 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index f93e330442b0..9c2d56a52428 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,10 +17,12 @@ name: Build And Deploy Playground Frontend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/frontend/**'] + paths: ['playground/backend/**'] workflow_dispatch: jobs: @@ -41,10 +43,18 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' + - name: Prepare Flutter lint + run: "flutter pub add flutter_lints --dev" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Lint - run: ./gradlew playground:frontend:analyze + run: "flutter analyze" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Tests - run: ./gradlew playground:frontend:test + run: flutter test + working-directory: playground/frontend/ + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -56,16 +66,12 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index 8846368ff8b6..d7275020dcfa 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE=apache/beam_java8_sdk:latest +ARG BASE_IMAGE FROM golang:1.17-buster AS build # Setup Go Environment From 2ef5533d006984046e5a509b1738f812137839bc Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Mon, 22 Nov 2021 12:11:43 +0300 Subject: [PATCH 3/4] Remove redundant methods from controller, move precompiled objects method to separate package --- playground/backend/cmd/server/controller.go | 335 +----------------- .../utils/precompiled_objects_utils.go | 23 ++ .../utils/precompiled_objects_utils_test.go | 60 ++++ 3 files changed, 87 insertions(+), 331 deletions(-) create mode 100644 playground/backend/internal/utils/precompiled_objects_utils.go create mode 100644 playground/backend/internal/utils/precompiled_objects_utils_test.go diff --git a/playground/backend/cmd/server/controller.go b/playground/backend/cmd/server/controller.go index 61b581a7ca6b..326971612cd0 100644 --- a/playground/backend/cmd/server/controller.go +++ b/playground/backend/cmd/server/controller.go @@ -160,14 +160,14 @@ func (controller *playgroundController) GetPrecompiledObjects(ctx context.Contex bucket := cloud_bucket.New() sdkToCategories, err := bucket.GetPrecompiledObjects(ctx, info.Sdk, info.Category) if err != nil { - logger.Errorf("%s: GetPrecompiledObjects(): cloud storage error: %s", err.Error()) + logger.Errorf("GetPrecompiledObjects(): cloud storage error: %s", err.Error()) return nil, errors.InternalError("GetPrecompiledObjects(): ", err.Error()) } response := pb.GetPrecompiledObjectsResponse{SdkCategories: make([]*pb.Categories, 0)} for sdkName, categories := range *sdkToCategories { sdkCategory := pb.Categories{Sdk: pb.Sdk(pb.Sdk_value[sdkName]), Categories: make([]*pb.Categories_Category, 0)} for categoryName, precompiledObjects := range categories { - PutPrecompiledObjectsToCategory(categoryName, &precompiledObjects, &sdkCategory) + utils.PutPrecompiledObjectsToCategory(categoryName, &precompiledObjects, &sdkCategory) } response.SdkCategories = append(response.SdkCategories, &sdkCategory) } @@ -179,7 +179,7 @@ func (controller *playgroundController) GetPrecompiledObjectCode(ctx context.Con cd := cloud_bucket.New() codeString, err := cd.GetPrecompiledObject(ctx, info.GetCloudPath()) if err != nil { - logger.Errorf("%s: GetPrecompiledObject(): cloud storage error: %s", err.Error()) + logger.Errorf("GetPrecompiledObject(): cloud storage error: %s", err.Error()) return nil, errors.InternalError("GetPrecompiledObjects(): ", err.Error()) } response := pb.GetPrecompiledObjectCodeResponse{Code: *codeString} @@ -191,336 +191,9 @@ func (controller *playgroundController) GetPrecompiledObjectOutput(ctx context.C cd := cloud_bucket.New() output, err := cd.GetPrecompiledObjectOutput(ctx, info.GetCloudPath()) if err != nil { - logger.Errorf("%s: GetPrecompiledObjectOutput(): cloud storage error: %s", err.Error()) + logger.Errorf("GetPrecompiledObjectOutput(): cloud storage error: %s", err.Error()) return nil, errors.InternalError("GetPrecompiledObjectOutput(): ", err.Error()) } response := pb.GetRunOutputResponse{Output: *output} return &response, nil } - -// PutPrecompiledObjectsToCategory adds categories with precompiled objects to protobuf object -func PutPrecompiledObjectsToCategory(categoryName string, precompiledObjects *cloud_bucket.PrecompiledObjects, sdkCategory *pb.Categories) { - category := pb.Categories_Category{ - CategoryName: categoryName, - PrecompiledObjects: make([]*pb.PrecompiledObject, 0), - } - for _, object := range *precompiledObjects { - category.PrecompiledObjects = append(category.PrecompiledObjects, &pb.PrecompiledObject{ - CloudPath: object.CloudPath, - Name: object.Name, - Description: object.Description, - Type: object.Type, - }) - } - sdkCategory.Categories = append(sdkCategory.Categories, &category) -} - -// setupLifeCycle creates fs_tool.LifeCycle and prepares files and folders needed to code processing -func setupLifeCycle(sdk pb.Sdk, code string, pipelineId uuid.UUID, workingDir string) (*fs_tool.LifeCycle, error) { - // create file system service - lc, err := fs_tool.NewLifeCycle(sdk, pipelineId, workingDir) - if err != nil { - logger.Errorf("%s: RunCode(): NewLifeCycle(): %s\n", pipelineId, err.Error()) - return nil, err - } - - // create folders - err = lc.CreateFolders() - if err != nil { - logger.Errorf("%s: RunCode(): CreateFolders(): %s\n", pipelineId, err.Error()) - return nil, err - } - - // create file with code - _, err = lc.CreateExecutableFile(code) - if err != nil { - logger.Errorf("%s: RunCode(): CreateExecutableFile(): %s\n", pipelineId, err.Error()) - return nil, err - } - return lc, nil -} - -// setupCompileBuilder returns executors.CompileBuilder with validators and compiler based on sdk -func setupCompileBuilder(lc *fs_tool.LifeCycle, sdk pb.Sdk, executorConfig *environment.ExecutorConfig) *executors.CompileBuilder { - filePath := lc.GetAbsoluteExecutableFilePath() - val := setupValidators(sdk, filePath) - - compileBuilder := executors.NewExecutorBuilder(). - WithValidator(). - WithSdkValidators(val). - WithCompiler() - - switch sdk { - case pb.Sdk_SDK_JAVA: - workingDir := lc.GetAbsoluteExecutableFilesFolderPath() - - compileBuilder = compileBuilder. - WithCommand(executorConfig.CompileCmd). - WithArgs(executorConfig.CompileArgs). - WithFileName(filePath). - WithWorkingDir(workingDir) - } - return compileBuilder -} - -// setupRunBuilder returns executors.RunBuilder based on sdk -func setupRunBuilder(pipelineId uuid.UUID, lc *fs_tool.LifeCycle, sdk pb.Sdk, env *environment.Environment, compileBuilder *executors.CompileBuilder) (*executors.RunBuilder, error) { - runBuilder := compileBuilder. - WithRunner(). - WithCommand(env.BeamSdkEnvs.ExecutorConfig.RunCmd). - WithArgs(env.BeamSdkEnvs.ExecutorConfig.RunArgs). - WithWorkingDir(lc.GetAbsoluteExecutableFilesFolderPath()) - - switch sdk { - case pb.Sdk_SDK_JAVA: - className, err := lc.ExecutableName(pipelineId, env.ApplicationEnvs.WorkingDir()) - if err != nil { - logger.Errorf("%s: get executable file name: %s\n", pipelineId, err.Error()) - return nil, err - } - - runBuilder = runBuilder. - WithClassName(className) - } - return runBuilder, nil -} - -// setupValidators returns slice of validators.Validator based on sdk -func setupValidators(sdk pb.Sdk, filepath string) *[]validators.Validator { - var val *[]validators.Validator - switch sdk { - case pb.Sdk_SDK_JAVA: - val = validators.GetJavaValidators(filepath) - } - return val -} - -// processCode validates, compiles and runs code by pipelineId. -// During each operation updates status of execution and saves it into cache: -// - In case of processing works more that timeout duration saves playground.Status_STATUS_RUN_TIMEOUT as cache.Status into cache. -// - In case of code processing has been canceled saves playground.Status_STATUS_CANCELED as cache.Status into cache. -// - In case of validation step is failed saves playground.Status_STATUS_VALIDATION_ERROR as cache.Status into cache. -// - In case of compile step is failed saves playground.Status_STATUS_COMPILE_ERROR as cache.Status and compile logs as cache.CompileOutput into cache. -// - In case of compile step is completed with no errors saves compile output as cache.CompileOutput into cache. -// - In case of run step is failed saves playground.Status_STATUS_RUN_ERROR as cache.Status and run logs as cache.RunError into cache. -// - In case of run step is completed with no errors saves playground.Status_STATUS_FINISHED as cache.Status and run output as cache.RunOutput into cache. -// At the end of this method deletes all created folders. -func processCode(ctx context.Context, cacheService cache.Cache, lc *fs_tool.LifeCycle, compileBuilder *executors.CompileBuilder, pipelineId uuid.UUID, env *environment.Environment, sdk pb.Sdk) { - ctxWithTimeout, finishCtxFunc := context.WithTimeout(ctx, env.ApplicationEnvs.PipelineExecuteTimeout()) - defer func(lc *fs_tool.LifeCycle) { - finishCtxFunc() - cleanUp(pipelineId, lc) - }(lc) - - errorChannel := make(chan error, 1) - dataChannel := make(chan interface{}, 1) - successChannel := make(chan bool, 1) - cancelChannel := make(chan bool, 1) - - go cancelCheck(ctxWithTimeout, pipelineId, cancelChannel, cacheService) - - // build executor for validate and compile steps - exec := compileBuilder.Build() - - // validate - logger.Infof("%s: Validate() ...\n", pipelineId) - validateFunc := exec.Validate() - go validateFunc(successChannel, errorChannel) - - if !processStep(ctxWithTimeout, pipelineId, cacheService, cancelChannel, successChannel, nil, errorChannel, pb.Status_STATUS_VALIDATION_ERROR, pb.Status_STATUS_PREPARING) { - return - } - - // prepare - logger.Infof("%s: Prepare() ...\n", pipelineId) - prepareFunc := exec.Prepare() - go prepareFunc(successChannel, errorChannel) - - if !processStep(ctxWithTimeout, pipelineId, cacheService, cancelChannel, successChannel, nil, errorChannel, pb.Status_STATUS_PREPARATION_ERROR, pb.Status_STATUS_COMPILING) { - return - } - - // compile - logger.Infof("%s: Compile() ...\n", pipelineId) - compileCmd := exec.Compile(ctxWithTimeout) - go func(successCh chan bool, errCh chan error, dataCh chan interface{}) { - // TODO separate stderr from stdout - data, err := compileCmd.CombinedOutput() - dataCh <- data - if err != nil { - errCh <- err - successCh <- false - } else { - successCh <- true - } - }(successChannel, errorChannel, dataChannel) - - if !processStep(ctxWithTimeout, pipelineId, cacheService, cancelChannel, successChannel, dataChannel, errorChannel, pb.Status_STATUS_COMPILE_ERROR, pb.Status_STATUS_EXECUTING) { - return - } - - runBuilder, err := setupRunBuilder(pipelineId, lc, sdk, env, compileBuilder) - if err != nil { - logger.Errorf("%s: error during setup runBuilder: %s\n", pipelineId, err.Error()) - setToCache(ctxWithTimeout, cacheService, pipelineId, cache.Status, pb.Status_STATUS_ERROR) - return - } - - // build executor for run step - exec = runBuilder.Build() - - // run - logger.Infof("%s: Run() ...\n", pipelineId) - runCmd := exec.Run(ctxWithTimeout) - go func(successCh chan bool, errCh chan error, dataCh chan interface{}) { - // TODO separate stderr from stdout - data, err := runCmd.CombinedOutput() - dataCh <- data - if err != nil { - errCh <- err - successChannel <- false - } else { - successChannel <- true - } - }(successChannel, errorChannel, dataChannel) - - processStep(ctxWithTimeout, pipelineId, cacheService, cancelChannel, successChannel, dataChannel, errorChannel, pb.Status_STATUS_RUN_ERROR, pb.Status_STATUS_FINISHED) -} - -// processStep processes each executor's step with cancel and timeout checks. -// If finishes by canceling, timeout or error - returns false. -// If finishes successfully returns true. -func processStep(ctx context.Context, pipelineId uuid.UUID, cacheService cache.Cache, cancelChannel, successChannel chan bool, dataChannel chan interface{}, errorChannel chan error, errorCaseStatus, successCaseStatus pb.Status) bool { - select { - case <-ctx.Done(): - finishByTimeout(ctx, pipelineId, cacheService) - return false - case <-cancelChannel: - processCancel(ctx, cacheService, pipelineId) - return false - case ok := <-successChannel: - var data []byte = nil - if dataChannel != nil { - temp := <-dataChannel - data = temp.([]byte) - } - if !ok { - err := <-errorChannel - processError(ctx, err, data, pipelineId, cacheService, errorCaseStatus) - return false - } - processSuccess(ctx, data, pipelineId, cacheService, successCaseStatus) - } - return true -} - -// finishByTimeout is used in case of runCode method finished by timeout -func finishByTimeout(ctx context.Context, pipelineId uuid.UUID, cacheService cache.Cache) { - logger.Errorf("%s: processCode finish because of timeout\n", pipelineId) - - // set to cache pipelineId: cache.SubKey_Status: Status_STATUS_RUN_TIMEOUT - setToCache(ctx, cacheService, pipelineId, cache.Status, pb.Status_STATUS_RUN_TIMEOUT) -} - -// cancelCheck checks cancel flag for code processing. -// If cancel flag doesn't exist in cache continue working. -// If context is done it means that code processing was finished (successfully/with error/timeout). Return. -// If cancel flag exists, and it is true it means that code processing was canceled. Set true to cancelChannel and return. -func cancelCheck(ctx context.Context, pipelineId uuid.UUID, cancelChannel chan bool, cacheService cache.Cache) { - ticker := time.NewTicker(500 * time.Millisecond) - for { - select { - case <-ctx.Done(): - ticker.Stop() - return - case _ = <-ticker.C: - cancel, err := cacheService.GetValue(ctx, pipelineId, cache.Canceled) - if err != nil { - continue - } - if cancel.(bool) { - cancelChannel <- true - } - return - } - } -} - -// cleanUp removes all prepared folders for received LifeCycle -func cleanUp(pipelineId uuid.UUID, lc *fs_tool.LifeCycle) { - logger.Infof("%s: DeleteFolders() ...\n", pipelineId) - if err := lc.DeleteFolders(); err != nil { - logger.Error("%s: DeleteFolders(): %s\n", pipelineId, err.Error()) - } - logger.Infof("%s: DeleteFolders() complete\n", pipelineId) - logger.Infof("%s: complete\n", pipelineId) -} - -// processError processes error received during processing code via setting a corresponding status and output to cache -func processError(ctx context.Context, err error, data []byte, pipelineId uuid.UUID, cacheService cache.Cache, status pb.Status) { - switch status { - case pb.Status_STATUS_VALIDATION_ERROR: - logger.Errorf("%s: Validate(): %s\n", pipelineId, err.Error()) - - setToCache(ctx, cacheService, pipelineId, cache.Status, pb.Status_STATUS_VALIDATION_ERROR) - case pb.Status_STATUS_PREPARATION_ERROR: - logger.Errorf("%s: Prepare(): %s\n", pipelineId, err.Error()) - - setToCache(ctx, cacheService, pipelineId, cache.Status, pb.Status_STATUS_PREPARATION_ERROR) - case pb.Status_STATUS_COMPILE_ERROR: - logger.Errorf("%s: Compile(): err: %s, output: %s\n", pipelineId, err.Error(), data) - - setToCache(ctx, cacheService, pipelineId, cache.CompileOutput, "error: "+err.Error()+", output: "+string(data)) - - setToCache(ctx, cacheService, pipelineId, cache.Status, pb.Status_STATUS_COMPILE_ERROR) - case pb.Status_STATUS_RUN_ERROR: - logger.Errorf("%s: Run(): err: %s, output: %s\n", pipelineId, err.Error(), data) - - setToCache(ctx, cacheService, pipelineId, cache.RunError, "error: "+err.Error()+", output: "+string(data)) - - setToCache(ctx, cacheService, pipelineId, cache.Status, pb.Status_STATUS_RUN_ERROR) - } -} - -// processSuccess processes case after successful code processing via setting a corresponding status and output to cache -func processSuccess(ctx context.Context, output []byte, pipelineId uuid.UUID, cacheService cache.Cache, status pb.Status) { - switch status { - case pb.Status_STATUS_PREPARING: - logger.Infof("%s: Validate() finish\n", pipelineId) - - setToCache(ctx, cacheService, pipelineId, cache.Status, pb.Status_STATUS_PREPARING) - case pb.Status_STATUS_COMPILING: - logger.Infof("%s: Prepare() finish\n", pipelineId) - - setToCache(ctx, cacheService, pipelineId, cache.Status, pb.Status_STATUS_COMPILING) - case pb.Status_STATUS_EXECUTING: - logger.Infof("%s: Compile() finish\n", pipelineId) - - setToCache(ctx, cacheService, pipelineId, cache.CompileOutput, string(output)) - - setToCache(ctx, cacheService, pipelineId, cache.Status, pb.Status_STATUS_EXECUTING) - case pb.Status_STATUS_FINISHED: - logger.Infof("%s: Run() finish\n", pipelineId) - - setToCache(ctx, cacheService, pipelineId, cache.RunOutput, string(output)) - - setToCache(ctx, cacheService, pipelineId, cache.Status, pb.Status_STATUS_FINISHED) - } -} - -// processCancel process case when code processing was canceled -func processCancel(ctx context.Context, cacheService cache.Cache, pipelineId uuid.UUID) { - logger.Infof("%s: was canceled\n", pipelineId) - - // set to cache pipelineId: cache.SubKey_Status: pb.Status_STATUS_CANCELED - setToCache(ctx, cacheService, pipelineId, cache.Status, pb.Status_STATUS_CANCELED) -} - -// setToCache puts value to cache by key and subKey -func setToCache(ctx context.Context, cacheService cache.Cache, key uuid.UUID, subKey cache.SubKey, value interface{}) error { - err := cacheService.SetValue(ctx, key, subKey, value) - if err != nil { - logger.Errorf("%s: cache.SetValue: %s\n", key, err.Error()) - } - return err -} diff --git a/playground/backend/internal/utils/precompiled_objects_utils.go b/playground/backend/internal/utils/precompiled_objects_utils.go new file mode 100644 index 000000000000..c1098e93fee2 --- /dev/null +++ b/playground/backend/internal/utils/precompiled_objects_utils.go @@ -0,0 +1,23 @@ +package utils + +import ( + pb "beam.apache.org/playground/backend/internal/api/v1" + "beam.apache.org/playground/backend/internal/cloud_bucket" +) + +// PutPrecompiledObjectsToCategory adds categories with precompiled objects to protobuf object +func PutPrecompiledObjectsToCategory(categoryName string, precompiledObjects *cloud_bucket.PrecompiledObjects, sdkCategory *pb.Categories) { + category := pb.Categories_Category{ + CategoryName: categoryName, + PrecompiledObjects: make([]*pb.PrecompiledObject, 0), + } + for _, object := range *precompiledObjects { + category.PrecompiledObjects = append(category.PrecompiledObjects, &pb.PrecompiledObject{ + CloudPath: object.CloudPath, + Name: object.Name, + Description: object.Description, + Type: object.Type, + }) + } + sdkCategory.Categories = append(sdkCategory.Categories, &category) +} diff --git a/playground/backend/internal/utils/precompiled_objects_utils_test.go b/playground/backend/internal/utils/precompiled_objects_utils_test.go new file mode 100644 index 000000000000..854df5f49ed9 --- /dev/null +++ b/playground/backend/internal/utils/precompiled_objects_utils_test.go @@ -0,0 +1,60 @@ +package utils + +import ( + pb "beam.apache.org/playground/backend/internal/api/v1" + "beam.apache.org/playground/backend/internal/cloud_bucket" + "reflect" + "testing" +) + +func TestPutPrecompiledObjectsToCategory(t *testing.T) { + precompiledObjectToAdd := &cloud_bucket.PrecompiledObjects{ + {"TestName", "SDK_JAVA/TestCategory/TestName.java", "TestDescription", pb.PrecompiledObjectType_PRECOMPILED_OBJECT_TYPE_EXAMPLE, []string{""}}, + } + type args struct { + categoryName string + precompiledObjects *cloud_bucket.PrecompiledObjects + sdkCategory *pb.Categories + } + tests := []struct { + name string + args args + want *pb.Categories + }{ + { + name: "Test PutPrecompiledObjectsToCategory", + args: args{ + categoryName: "TestCategory", + precompiledObjects: precompiledObjectToAdd, + sdkCategory: &pb.Categories{ + Sdk: pb.Sdk_SDK_JAVA, + Categories: []*pb.Categories_Category{}, + }, + }, + want: &pb.Categories{ + Sdk: pb.Sdk_SDK_JAVA, + Categories: []*pb.Categories_Category{ + { + CategoryName: "TestCategory", PrecompiledObjects: []*pb.PrecompiledObject{ + { + CloudPath: "SDK_JAVA/TestCategory/TestName.java", + Name: "TestName", + Description: "TestDescription", + Type: pb.PrecompiledObjectType_PRECOMPILED_OBJECT_TYPE_EXAMPLE, + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + PutPrecompiledObjectsToCategory(tt.args.categoryName, tt.args.precompiledObjects, tt.args.sdkCategory) + got := tt.args.sdkCategory + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("PutPrecompiledObjectsToCategory() got = %v, want %v", got, tt.want) + } + }) + } +} From 6488340c30bedf3a5d921b24b883b22142f5586f Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Mon, 22 Nov 2021 13:50:14 +0300 Subject: [PATCH 4/4] fix RAT --- .../internal/utils/precompiled_objects_utils.go | 15 +++++++++++++++ .../utils/precompiled_objects_utils_test.go | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/playground/backend/internal/utils/precompiled_objects_utils.go b/playground/backend/internal/utils/precompiled_objects_utils.go index c1098e93fee2..3abb46862516 100644 --- a/playground/backend/internal/utils/precompiled_objects_utils.go +++ b/playground/backend/internal/utils/precompiled_objects_utils.go @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils import ( diff --git a/playground/backend/internal/utils/precompiled_objects_utils_test.go b/playground/backend/internal/utils/precompiled_objects_utils_test.go index 854df5f49ed9..07a7cbc03bb3 100644 --- a/playground/backend/internal/utils/precompiled_objects_utils_test.go +++ b/playground/backend/internal/utils/precompiled_objects_utils_test.go @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils import (