From b73f5f70ac4184b56a0a03922731c5f2f69b9566 Mon Sep 17 00:00:00 2001 From: Sergey Kalinin <91209855+snkalinin@users.noreply.github.com> Date: Tue, 16 Nov 2021 14:59:11 +0300 Subject: [PATCH 01/12] [Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83) * Update workflows for playground * Attempt to fix tests * Remove continue on error to catch errors * Fix linter problem for backend dockerfile * Update folder to run backend go linter * Moved flutter test to execution via gradle tasks --- .../workflows/build_playground_backend.yml | 14 ++++++++----- .../workflows/build_playground_frontend.yml | 20 +++++++------------ playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index 96e8aae9b0d5..bd2868768246 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,10 +17,8 @@ name: Build And Deploy Playground Backend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -45,12 +43,14 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint - run: "golangci-lint run internal/..." +# run: "golangci-lint run internal/..." + run: "golangci-lint run cmd/server/..." working-directory: playground/backend/ - continue-on-error: true + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue + run: rm ~/.m2/settings.xml - name: Run Tests run: ./gradlew playground:backend:test - continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,12 +62,16 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index 9c2d56a52428..f93e330442b0 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,12 +17,10 @@ name: Build And Deploy Playground Frontend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/backend/**'] + paths: ['playground/frontend/**'] workflow_dispatch: jobs: @@ -43,18 +41,10 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' - - name: Prepare Flutter lint - run: "flutter pub add flutter_lints --dev" - working-directory: playground/frontend/ - continue-on-error: true - name: Run Lint - run: "flutter analyze" - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:analyze - name: Run Tests - run: flutter test - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:test - name: install npm uses: actions/setup-node@v2 with: @@ -66,12 +56,16 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index d7275020dcfa..8846368ff8b6 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE +ARG BASE_IMAGE=apache/beam_java8_sdk:latest FROM golang:1.17-buster AS build # Setup Go Environment From d449275b8ebd623760b34981f56c8f62b140ad9d Mon Sep 17 00:00:00 2001 From: Ilya Date: Tue, 16 Nov 2021 15:07:49 +0300 Subject: [PATCH 02/12] Revert "[Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83)" (#88) This reverts commit b73f5f70ac4184b56a0a03922731c5f2f69b9566. --- .../workflows/build_playground_backend.yml | 14 +++++-------- .../workflows/build_playground_frontend.yml | 20 ++++++++++++------- playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index bd2868768246..96e8aae9b0d5 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,8 +17,10 @@ name: Build And Deploy Playground Backend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -43,14 +45,12 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint -# run: "golangci-lint run internal/..." - run: "golangci-lint run cmd/server/..." + run: "golangci-lint run internal/..." working-directory: playground/backend/ - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: rm ~/.m2/settings.xml + continue-on-error: true - name: Run Tests run: ./gradlew playground:backend:test + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,16 +62,12 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index f93e330442b0..9c2d56a52428 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,10 +17,12 @@ name: Build And Deploy Playground Frontend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/frontend/**'] + paths: ['playground/backend/**'] workflow_dispatch: jobs: @@ -41,10 +43,18 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' + - name: Prepare Flutter lint + run: "flutter pub add flutter_lints --dev" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Lint - run: ./gradlew playground:frontend:analyze + run: "flutter analyze" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Tests - run: ./gradlew playground:frontend:test + run: flutter test + working-directory: playground/frontend/ + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -56,16 +66,12 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index 8846368ff8b6..d7275020dcfa 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE=apache/beam_java8_sdk:latest +ARG BASE_IMAGE FROM golang:1.17-buster AS build # Setup Go Environment From 6b6f0b98392c6fa68e384cf260589f689c611614 Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Fri, 19 Nov 2021 18:08:57 +0300 Subject: [PATCH 03/12] draft --- playground/backend/cmd/server/controller.go | 2 +- playground/backend/configs/SDK_PYTHON.json | 6 ++ .../code_processing/code_processing.go | 18 +++-- .../environment/environment_service.go | 2 +- .../backend/internal/fs_tool/compiling_lc.go | 18 ++++- playground/backend/internal/fs_tool/fs.go | 2 + .../backend/internal/fs_tool/python_fs.go | 29 +++++++ .../internal/fs_tool/python_fs_test.go | 76 +++++++++++++++++++ .../run_builder/run_builder_setuper.go | 2 + .../internal/utils/validators_utils.go | 2 + .../internal/validators/python_validators.go | 22 ++++++ 11 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 playground/backend/configs/SDK_PYTHON.json create mode 100644 playground/backend/internal/fs_tool/python_fs.go create mode 100644 playground/backend/internal/fs_tool/python_fs_test.go create mode 100644 playground/backend/internal/validators/python_validators.go diff --git a/playground/backend/cmd/server/controller.go b/playground/backend/cmd/server/controller.go index 9c578b2189d3..a553684b8a98 100644 --- a/playground/backend/cmd/server/controller.go +++ b/playground/backend/cmd/server/controller.go @@ -50,7 +50,7 @@ func (controller *playgroundController) RunCode(ctx context.Context, info *pb.Ru return nil, errors.InvalidArgumentError("Run code()", "incorrect sdk: "+info.Sdk.String()) } switch info.Sdk { - case pb.Sdk_SDK_UNSPECIFIED, pb.Sdk_SDK_PYTHON, pb.Sdk_SDK_SCIO: + case pb.Sdk_SDK_UNSPECIFIED, pb.Sdk_SDK_SCIO: logger.Errorf("RunCode(): unimplemented sdk: %s\n", info.Sdk) return nil, errors.InvalidArgumentError("Run code()", "unimplemented sdk: "+info.Sdk.String()) } diff --git a/playground/backend/configs/SDK_PYTHON.json b/playground/backend/configs/SDK_PYTHON.json new file mode 100644 index 000000000000..ddb781d6f354 --- /dev/null +++ b/playground/backend/configs/SDK_PYTHON.json @@ -0,0 +1,6 @@ +{ + "compile_cmd": "", + "run_cmd": "python", + "compile_args": [], + "run_args": [] +} \ No newline at end of file diff --git a/playground/backend/internal/code_processing/code_processing.go b/playground/backend/internal/code_processing/code_processing.go index d68980286908..5f3cb2feaca8 100644 --- a/playground/backend/internal/code_processing/code_processing.go +++ b/playground/backend/internal/code_processing/code_processing.go @@ -76,13 +76,17 @@ func Process(ctx context.Context, cacheService cache.Cache, lc *fs_tool.LifeCycl return } - // compile - logger.Infof("%s: Compile() ...\n", pipelineId) - compileCmd := exec.Compile(ctxWithTimeout) - go processCmd(compileCmd, successChannel, errorChannel, dataChannel) - - if err := processStep(ctxWithTimeout, pipelineId, cacheService, cancelChannel, successChannel, dataChannel, errorChannel, pb.Status_STATUS_COMPILE_ERROR, pb.Status_STATUS_EXECUTING); err != nil { - return + switch sdkEnv.ApacheBeamSdk { + case pb.Sdk_SDK_JAVA: + case pb.Sdk_SDK_GO: + // compile + logger.Infof("%s: Compile() ...\n", pipelineId) + compileCmd := exec.Compile(ctxWithTimeout) + go processCmd(compileCmd, successChannel, errorChannel, dataChannel) + + if err := processStep(ctxWithTimeout, pipelineId, cacheService, cancelChannel, successChannel, dataChannel, errorChannel, pb.Status_STATUS_COMPILE_ERROR, pb.Status_STATUS_EXECUTING); err != nil { + return + } } runBuilder, err := run_builder.Setup(pipelineId, lc, appEnv.WorkingDir(), sdkEnv, compileBuilder) diff --git a/playground/backend/internal/environment/environment_service.go b/playground/backend/internal/environment/environment_service.go index 78b90b548d03..941bb54fcb2c 100644 --- a/playground/backend/internal/environment/environment_service.go +++ b/playground/backend/internal/environment/environment_service.go @@ -182,7 +182,7 @@ func createExecutorConfig(apacheBeamSdk pb.Sdk, configPath string) (*ExecutorCon case pb.Sdk_SDK_GO: // Go sdk doesn't need any additional arguments from the config file case pb.Sdk_SDK_PYTHON: - return nil, errors.New("not yet supported") + // Python sdk doesn't need any additional arguments from the config file case pb.Sdk_SDK_SCIO: return nil, errors.New("not yet supported") } diff --git a/playground/backend/internal/fs_tool/compiling_lc.go b/playground/backend/internal/fs_tool/compiling_lc.go index 94136eda2dc5..85a5cfadc584 100644 --- a/playground/backend/internal/fs_tool/compiling_lc.go +++ b/playground/backend/internal/fs_tool/compiling_lc.go @@ -26,7 +26,7 @@ const ( compiledFolderName = "bin" ) -// newCompilingLifeCycle creates LifeCycle with compiled SDK environment. +// newCompilingLifeCycle creates LifeCycle for compiled SDK environment. func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, executableFileExtension string, compiledFileExtension string) *LifeCycle { baseFileFolder := filepath.Join(workingDir, baseFileFolder, pipelineId.String()) srcFileFolder := filepath.Join(baseFileFolder, sourceFolderName) @@ -45,3 +45,19 @@ func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, executableFi pipelineId: pipelineId, } } + +// newInterpretedLifeCycle creates LifeCycle for interpreted SDK environment. +func newInterpretedLifeCycle(pipelineId uuid.UUID, workingDir string, executableFileExtension string) *LifeCycle { + executableFileFolder := filepath.Join(workingDir, baseFileFolder, pipelineId.String()) + return &LifeCycle{ + folderGlobs: []string{executableFileFolder}, + Folder: Folder{ + BaseFolder: executableFileFolder, + ExecutableFolder: executableFileFolder, + }, + Extension: Extension{ + ExecutableExtension: executableFileExtension, + }, + pipelineId: pipelineId, + } +} diff --git a/playground/backend/internal/fs_tool/fs.go b/playground/backend/internal/fs_tool/fs.go index 77ba7e72ef0a..e04974a7272c 100644 --- a/playground/backend/internal/fs_tool/fs.go +++ b/playground/backend/internal/fs_tool/fs.go @@ -61,6 +61,8 @@ func NewLifeCycle(sdk pb.Sdk, pipelineId uuid.UUID, workingDir string) (*LifeCyc return newJavaLifeCycle(pipelineId, workingDir), nil case pb.Sdk_SDK_GO: return newGoLifeCycle(pipelineId, workingDir), nil + case pb.Sdk_SDK_PYTHON: + return newPythonLifeCycle(pipelineId, workingDir), nil default: return nil, fmt.Errorf("%s isn't supported now", sdk) } diff --git a/playground/backend/internal/fs_tool/python_fs.go b/playground/backend/internal/fs_tool/python_fs.go new file mode 100644 index 000000000000..4fc35241a582 --- /dev/null +++ b/playground/backend/internal/fs_tool/python_fs.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_tool + +import ( + "github.com/google/uuid" +) + +const ( + pythonExecutableFileExtension = ".py" +) + +// newPythonLifeCycle creates LifeCycle with go SDK environment. +func newPythonLifeCycle(pipelineId uuid.UUID, workingDir string) *LifeCycle { + return newInterpretedLifeCycle(pipelineId, workingDir, pythonExecutableFileExtension) +} diff --git a/playground/backend/internal/fs_tool/python_fs_test.go b/playground/backend/internal/fs_tool/python_fs_test.go new file mode 100644 index 000000000000..f3f0515fc940 --- /dev/null +++ b/playground/backend/internal/fs_tool/python_fs_test.go @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs_tool + +import ( + "fmt" + "github.com/google/uuid" + "reflect" + "testing" +) + +func Test_newPythonLifeCycle(t *testing.T) { + pipelineId := uuid.New() + workingDir := "workingDir" + baseFileFolder := fmt.Sprintf("%s/%s/%s", workingDir, baseFileFolder, pipelineId) + + type args struct { + pipelineId uuid.UUID + workingDir string + } + tests := []struct { + name string + args args + want *LifeCycle + }{ + { + // Test case with calling newGoLifeCycle method with correct pipelineId and workingDir. + // As a result, want to receive an expected go life cycle. + name: "newPythonLifeCycle", + args: args{ + pipelineId: pipelineId, + workingDir: workingDir, + }, + want: &LifeCycle{ + folderGlobs: []string{baseFileFolder}, + Folder: Folder{ + BaseFolder: baseFileFolder, + }, + Extension: Extension{ + ExecutableExtension: pythonExecutableFileExtension, + }, + pipelineId: pipelineId, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := newPythonLifeCycle(tt.args.pipelineId, tt.args.workingDir) + if !reflect.DeepEqual(got.folderGlobs, tt.want.folderGlobs) { + t.Errorf("newPythonLifeCycle() folderGlobs = %v, want %v", got.folderGlobs, tt.want.folderGlobs) + } + if !reflect.DeepEqual(got.Folder, tt.want.Folder) { + t.Errorf("newPythonLifeCycle() Folder = %v, want %v", got.Folder, tt.want.Folder) + } + if !reflect.DeepEqual(got.Extension, tt.want.Extension) { + t.Errorf("newPythonLifeCycle() Extension = %v, want %v", got.Extension, tt.want.Extension) + } + if !reflect.DeepEqual(got.pipelineId, tt.want.pipelineId) { + t.Errorf("newPythonLifeCycle() pipelineId = %v, want %v", got.pipelineId, tt.want.pipelineId) + } + }) + } +} diff --git a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go index 09bea69437af..ad98a2eea7f5 100644 --- a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go +++ b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go @@ -45,6 +45,8 @@ func Setup(pipelineId uuid.UUID, lc *fs_tool.LifeCycle, workingDir string, sdkEn WithClassName(className) case pb.Sdk_SDK_GO: runBuilder = runBuilder.WithCommand(lc.GetAbsoluteBinaryFilePath()) + case pb.Sdk_SDK_PYTHON: + runBuilder = runBuilder.WithCommand(lc.GetAbsoluteBinaryFilePath()) default: return nil, fmt.Errorf("incorrect sdk: %s", sdkEnv.ApacheBeamSdk) } diff --git a/playground/backend/internal/utils/validators_utils.go b/playground/backend/internal/utils/validators_utils.go index c5e576177830..8cbe367c9b4b 100644 --- a/playground/backend/internal/utils/validators_utils.go +++ b/playground/backend/internal/utils/validators_utils.go @@ -29,6 +29,8 @@ func GetValidators(sdk pb.Sdk, filepath string) (*[]validators.Validator, error) val = validators.GetJavaValidators(filepath) case pb.Sdk_SDK_GO: val = validators.GetGoValidators() + case pb.Sdk_SDK_PYTHON: + val = validators.GetPythonValidators() default: return nil, fmt.Errorf("incorrect sdk: %s", sdk) } diff --git a/playground/backend/internal/validators/python_validators.go b/playground/backend/internal/validators/python_validators.go new file mode 100644 index 000000000000..4d997337df78 --- /dev/null +++ b/playground/backend/internal/validators/python_validators.go @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validators + +// GetPythonValidators return validators methods that should be applied to Go code +func GetPythonValidators() *[]Validator { + //TODO: Will be added in task [BEAM-13292] + return &[]Validator{} +} From df5e0530db506305702d8a3b812fa162e64e4c09 Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Mon, 22 Nov 2021 11:15:39 +0300 Subject: [PATCH 04/12] Rename LF executable -> source --- playground/backend/cmd/server/controller.go | 2 +- .../code_processing/code_processing_test.go | 6 ++-- playground/backend/internal/fs_tool/fs.go | 22 ++++++------- .../backend/internal/fs_tool/fs_test.go | 32 +++++++++---------- .../backend/internal/fs_tool/go_fs_test.go | 4 +-- .../backend/internal/fs_tool/java_fs.go | 6 ++-- .../backend/internal/fs_tool/java_fs_test.go | 4 +-- .../{compiling_lc.go => lc_constructor.go} | 8 ++--- .../preparators/java_preparators_test.go | 4 +-- .../life_cycle/life_cycle_setuper.go | 4 +-- 10 files changed, 46 insertions(+), 46 deletions(-) rename playground/backend/internal/fs_tool/{compiling_lc.go => lc_constructor.go} (86%) diff --git a/playground/backend/cmd/server/controller.go b/playground/backend/cmd/server/controller.go index 61b581a7ca6b..5f63e6b56558 100644 --- a/playground/backend/cmd/server/controller.go +++ b/playground/backend/cmd/server/controller.go @@ -65,7 +65,7 @@ func (controller *playgroundController) RunCode(ctx context.Context, info *pb.Ru return nil, errors.InternalError("Run code", "Error during setup file system: "+err.Error()) } - compileBuilder, err := compile_builder.Setup(lc.GetAbsoluteExecutableFilePath(), lc.GetAbsoluteExecutableFilesFolderPath(), info.Sdk, controller.env.BeamSdkEnvs.ExecutorConfig) + compileBuilder, err := compile_builder.Setup(lc.GetAbsoluteSourceFilePath(), lc.GetAbsoluteExecutableFilesFolderPath(), info.Sdk, controller.env.BeamSdkEnvs.ExecutorConfig) if err != nil { logger.Errorf("RunCode(): error during setup run builder: %s\n", err.Error()) return nil, errors.InvalidArgumentError("Run code", "Error during setup compile builder: "+err.Error()) diff --git a/playground/backend/internal/code_processing/code_processing_test.go b/playground/backend/internal/code_processing/code_processing_test.go index 1fe0ea312d94..5f0fb2ba8af2 100644 --- a/playground/backend/internal/code_processing/code_processing_test.go +++ b/playground/backend/internal/code_processing/code_processing_test.go @@ -222,7 +222,7 @@ func Test_Process(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { lc, _ := fs_tool.NewLifeCycle(pb.Sdk_SDK_JAVA, tt.args.pipelineId, os.Getenv("APP_WORK_DIR")) - filePath := lc.GetAbsoluteExecutableFilePath() + filePath := lc.GetAbsoluteSourceFilePath() workingDir := lc.GetAbsoluteExecutableFilesFolderPath() exec := executors.NewExecutorBuilder(). @@ -239,7 +239,7 @@ func Test_Process(t *testing.T) { t.Fatalf("error during prepare folders: %s", err.Error()) } if tt.createExecFile { - _, _ = lc.CreateExecutableFile(tt.code) + _, _ = lc.CreateSourceCodeFile(tt.code) } if tt.cancelFunc { @@ -258,7 +258,7 @@ func Test_Process(t *testing.T) { compileOutput, _ := cacheService.GetValue(tt.args.ctx, tt.args.pipelineId, cache.CompileOutput) if tt.expectedCompileOutput != nil && strings.Contains(tt.expectedCompileOutput.(string), "%s") { - tt.expectedCompileOutput = fmt.Sprintf(tt.expectedCompileOutput.(string), lc.GetAbsoluteExecutableFilePath()) + tt.expectedCompileOutput = fmt.Sprintf(tt.expectedCompileOutput.(string), lc.GetAbsoluteSourceFilePath()) } if !reflect.DeepEqual(compileOutput, tt.expectedCompileOutput) { t.Errorf("processCode() set compileOutput: %s, but expectes: %s", compileOutput, tt.expectedCompileOutput) diff --git a/playground/backend/internal/fs_tool/fs.go b/playground/backend/internal/fs_tool/fs.go index 77ba7e72ef0a..5e7ceaa22295 100644 --- a/playground/backend/internal/fs_tool/fs.go +++ b/playground/backend/internal/fs_tool/fs.go @@ -32,14 +32,14 @@ const ( // For each SDK these values should be set depending on folders that need for the SDK. type Folder struct { BaseFolder string - ExecutableFolder string + SourceFileFolder string CompiledFolder string } // Extension contains executable and compiled files' extensions. // For each SDK these values should be set depending on SDK's extensions. type Extension struct { - ExecutableExtension string + SourceFileExtension string CompiledExtension string } @@ -88,14 +88,14 @@ func (l *LifeCycle) DeleteFolders() error { return nil } -// CreateExecutableFile creates an executable file (i.e. file.{executableExtension}). -func (l *LifeCycle) CreateExecutableFile(code string) (string, error) { - if _, err := os.Stat(l.Folder.ExecutableFolder); os.IsNotExist(err) { +// CreateSourceCodeFile creates an executable file (i.e. file.{sourceFileExtension}). +func (l *LifeCycle) CreateSourceCodeFile(code string) (string, error) { + if _, err := os.Stat(l.Folder.SourceFileFolder); os.IsNotExist(err) { return "", err } - fileName := l.pipelineId.String() + l.Extension.ExecutableExtension - filePath := filepath.Join(l.Folder.ExecutableFolder, fileName) + fileName := l.pipelineId.String() + l.Extension.SourceFileExtension + filePath := filepath.Join(l.Folder.SourceFileFolder, fileName) err := os.WriteFile(filePath, []byte(code), fileMode) if err != nil { return "", err @@ -103,10 +103,10 @@ func (l *LifeCycle) CreateExecutableFile(code string) (string, error) { return fileName, nil } -// GetAbsoluteExecutableFilePath returns absolute filepath to executable file (/path/to/workingDir/executable_files/{pipelineId}/src/{pipelineId}.{executableExtension}). -func (l *LifeCycle) GetAbsoluteExecutableFilePath() string { - fileName := l.pipelineId.String() + l.Extension.ExecutableExtension - filePath := filepath.Join(l.Folder.ExecutableFolder, fileName) +// GetAbsoluteSourceFilePath returns absolute filepath to executable file (/path/to/workingDir/executable_files/{pipelineId}/src/{pipelineId}.{sourceFileExtension}). +func (l *LifeCycle) GetAbsoluteSourceFilePath() string { + fileName := l.pipelineId.String() + l.Extension.SourceFileExtension + filePath := filepath.Join(l.Folder.SourceFileFolder, fileName) absoluteFilePath, _ := filepath.Abs(filePath) return absoluteFilePath } diff --git a/playground/backend/internal/fs_tool/fs_test.go b/playground/backend/internal/fs_tool/fs_test.go index 9e9f66c691cf..f87664fa9090 100644 --- a/playground/backend/internal/fs_tool/fs_test.go +++ b/playground/backend/internal/fs_tool/fs_test.go @@ -53,7 +53,7 @@ func TestLifeCycle_CreateExecutableFile(t *testing.T) { name: "executable folder doesn't exist", fields: fields{ folder: Folder{ - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, CompiledFolder: binFileFolder, }, pipelineId: pipelineId, @@ -66,12 +66,12 @@ func TestLifeCycle_CreateExecutableFile(t *testing.T) { name: "executable folder exists", createFolders: []string{srcFileFolder}, fields: fields{ - folder: Folder{ExecutableFolder: srcFileFolder}, - extension: Extension{ExecutableExtension: javaExecutableFileExtension}, + folder: Folder{SourceFileFolder: srcFileFolder}, + extension: Extension{SourceFileExtension: javaSourceFileExtension}, pipelineId: pipelineId, }, args: args{code: "TEST_CODE"}, - want: pipelineId.String() + javaExecutableFileExtension, + want: pipelineId.String() + javaSourceFileExtension, wantErr: false, }, } @@ -86,13 +86,13 @@ func TestLifeCycle_CreateExecutableFile(t *testing.T) { Extension: tt.fields.extension, pipelineId: tt.fields.pipelineId, } - got, err := l.CreateExecutableFile(tt.args.code) + got, err := l.CreateSourceCodeFile(tt.args.code) if (err != nil) != tt.wantErr { - t.Errorf("CreateExecutableFile() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("CreateSourceCodeFile() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("CreateExecutableFile() got = %v, want %v", got, tt.want) + t.Errorf("CreateSourceCodeFile() got = %v, want %v", got, tt.want) } }) os.RemoveAll(baseFileFolder) @@ -209,11 +209,11 @@ func TestNewLifeCycle(t *testing.T) { folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ BaseFolder: baseFileFolder, - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, CompiledFolder: binFileFolder, }, Extension: Extension{ - ExecutableExtension: javaExecutableFileExtension, + SourceFileExtension: javaSourceFileExtension, CompiledExtension: javaCompiledFileExtension, }, ExecutableName: executableName, @@ -260,7 +260,7 @@ func TestLifeCycle_GetAbsoluteExecutableFilePath(t *testing.T) { baseFileFolder := fmt.Sprintf("%s_%s", baseFileFolder, pipelineId) srcFileFolder := baseFileFolder + "/src" - filePath := fmt.Sprintf("%s/%s", srcFileFolder, pipelineId.String()+javaExecutableFileExtension) + filePath := fmt.Sprintf("%s/%s", srcFileFolder, pipelineId.String()+javaSourceFileExtension) absolutePath, _ := filepath.Abs(filePath) type fields struct { folderGlobs []string @@ -275,13 +275,13 @@ func TestLifeCycle_GetAbsoluteExecutableFilePath(t *testing.T) { wantErr bool }{ { - name: "GetAbsoluteExecutableFilePath", + name: "GetAbsoluteSourceFilePath", fields: fields{ Folder: Folder{ BaseFolder: baseFileFolder, - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, }, - Extension: Extension{ExecutableExtension: javaExecutableFileExtension}, + Extension: Extension{SourceFileExtension: javaSourceFileExtension}, pipelineId: pipelineId, }, want: absolutePath, @@ -295,9 +295,9 @@ func TestLifeCycle_GetAbsoluteExecutableFilePath(t *testing.T) { Extension: tt.fields.Extension, pipelineId: tt.fields.pipelineId, } - got := l.GetAbsoluteExecutableFilePath() + got := l.GetAbsoluteSourceFilePath() if got != tt.want { - t.Errorf("GetAbsoluteExecutableFilePath() got = %v, want %v", got, tt.want) + t.Errorf("GetAbsoluteSourceFilePath() got = %v, want %v", got, tt.want) } }) } @@ -324,7 +324,7 @@ func TestLifeCycle_GetAbsoluteExecutableFilesFolderPath(t *testing.T) { name: "GetAbsoluteExecutableFolderPath", fields: fields{ Folder: Folder{BaseFolder: baseFileFolder}, - Extension: Extension{ExecutableExtension: javaExecutableFileExtension}, + Extension: Extension{SourceFileExtension: javaSourceFileExtension}, pipelineId: pipelineId, }, want: absolutePath, diff --git a/playground/backend/internal/fs_tool/go_fs_test.go b/playground/backend/internal/fs_tool/go_fs_test.go index f8925ce5d32f..433d0b7aafbf 100644 --- a/playground/backend/internal/fs_tool/go_fs_test.go +++ b/playground/backend/internal/fs_tool/go_fs_test.go @@ -50,11 +50,11 @@ func Test_newGoLifeCycle(t *testing.T) { folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ BaseFolder: baseFileFolder, - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, CompiledFolder: binFileFolder, }, Extension: Extension{ - ExecutableExtension: goExecutableFileExtension, + SourceFileExtension: goExecutableFileExtension, CompiledExtension: goCompiledFileExtension, }, pipelineId: pipelineId, diff --git a/playground/backend/internal/fs_tool/java_fs.go b/playground/backend/internal/fs_tool/java_fs.go index e89773040526..b2c81ccb7ab8 100644 --- a/playground/backend/internal/fs_tool/java_fs.go +++ b/playground/backend/internal/fs_tool/java_fs.go @@ -24,13 +24,13 @@ import ( ) const ( - javaExecutableFileExtension = ".java" - javaCompiledFileExtension = ".class" + javaSourceFileExtension = ".java" + javaCompiledFileExtension = ".class" ) // newJavaLifeCycle creates LifeCycle with java SDK environment. func newJavaLifeCycle(pipelineId uuid.UUID, workingDir string) *LifeCycle { - javaLifeCycle := newCompilingLifeCycle(pipelineId, workingDir, javaExecutableFileExtension, javaCompiledFileExtension) + javaLifeCycle := newCompilingLifeCycle(pipelineId, workingDir, javaSourceFileExtension, javaCompiledFileExtension) javaLifeCycle.ExecutableName = executableName return javaLifeCycle } diff --git a/playground/backend/internal/fs_tool/java_fs_test.go b/playground/backend/internal/fs_tool/java_fs_test.go index a65e4562cea0..175ea5fd19cf 100644 --- a/playground/backend/internal/fs_tool/java_fs_test.go +++ b/playground/backend/internal/fs_tool/java_fs_test.go @@ -52,11 +52,11 @@ func Test_newJavaLifeCycle(t *testing.T) { folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ BaseFolder: baseFileFolder, - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, CompiledFolder: binFileFolder, }, Extension: Extension{ - ExecutableExtension: javaExecutableFileExtension, + SourceFileExtension: javaSourceFileExtension, CompiledExtension: javaCompiledFileExtension, }, ExecutableName: executableName, diff --git a/playground/backend/internal/fs_tool/compiling_lc.go b/playground/backend/internal/fs_tool/lc_constructor.go similarity index 86% rename from playground/backend/internal/fs_tool/compiling_lc.go rename to playground/backend/internal/fs_tool/lc_constructor.go index 94136eda2dc5..dbadbca9beea 100644 --- a/playground/backend/internal/fs_tool/compiling_lc.go +++ b/playground/backend/internal/fs_tool/lc_constructor.go @@ -26,8 +26,8 @@ const ( compiledFolderName = "bin" ) -// newCompilingLifeCycle creates LifeCycle with compiled SDK environment. -func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, executableFileExtension string, compiledFileExtension string) *LifeCycle { +// newCompilingLifeCycle creates LifeCycle for compiled SDK environment. +func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, sourceFileExtension string, compiledFileExtension string) *LifeCycle { baseFileFolder := filepath.Join(workingDir, baseFileFolder, pipelineId.String()) srcFileFolder := filepath.Join(baseFileFolder, sourceFolderName) binFileFolder := filepath.Join(baseFileFolder, compiledFolderName) @@ -35,11 +35,11 @@ func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, executableFi folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ BaseFolder: baseFileFolder, - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, CompiledFolder: binFileFolder, }, Extension: Extension{ - ExecutableExtension: executableFileExtension, + SourceFileExtension: sourceFileExtension, CompiledExtension: compiledFileExtension, }, pipelineId: pipelineId, diff --git a/playground/backend/internal/preparators/java_preparators_test.go b/playground/backend/internal/preparators/java_preparators_test.go index a34579798829..7bfeef92c671 100644 --- a/playground/backend/internal/preparators/java_preparators_test.go +++ b/playground/backend/internal/preparators/java_preparators_test.go @@ -36,7 +36,7 @@ func Test_replace(t *testing.T) { lc, _ := fs_tool.NewLifeCycle(pb.Sdk_SDK_JAVA, uuid.New(), filepath.Join(path, "temp")) _ = lc.CreateFolders() defer os.RemoveAll(filepath.Join(path, "temp")) - _, _ = lc.CreateExecutableFile(codeWithPublicClass) + _, _ = lc.CreateSourceCodeFile(codeWithPublicClass) type args struct { args []interface{} @@ -54,7 +54,7 @@ func Test_replace(t *testing.T) { }, { name: "original file exists", - args: args{[]interface{}{lc.GetAbsoluteExecutableFilePath(), classWithPublicModifierPattern, classWithoutPublicModifierPattern}}, + args: args{[]interface{}{lc.GetAbsoluteSourceFilePath(), classWithPublicModifierPattern, classWithoutPublicModifierPattern}}, wantCode: codeWithoutPublicClass, wantErr: false, }, diff --git a/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go b/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go index 63ae04ad61de..b13d7bff6c28 100644 --- a/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go +++ b/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go @@ -40,9 +40,9 @@ func Setup(sdk pb.Sdk, code string, pipelineId uuid.UUID, workingDir string) (*f } // create file with code - _, err = lc.CreateExecutableFile(code) + _, err = lc.CreateSourceCodeFile(code) if err != nil { - logger.Errorf("%s: RunCode(): CreateExecutableFile(): %s\n", pipelineId, err.Error()) + logger.Errorf("%s: RunCode(): CreateSourceCodeFile(): %s\n", pipelineId, err.Error()) return nil, err } return lc, nil From 6b7603d16f17eafc63e7e2f5ff59f9cdd342b48f Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Mon, 22 Nov 2021 13:31:55 +0300 Subject: [PATCH 05/12] Reanme LF compile -> executables --- playground/backend/cmd/server/controller.go | 2 +- .../code_processing/code_processing_test.go | 2 +- .../internal/executors/executor_builder.go | 4 ++-- .../internal/executors/executor_test.go | 2 +- playground/backend/internal/fs_tool/fs.go | 22 +++++++++---------- .../backend/internal/fs_tool/fs_test.go | 22 +++++++++---------- playground/backend/internal/fs_tool/go_fs.go | 6 ++--- .../backend/internal/fs_tool/go_fs_test.go | 10 ++++----- .../backend/internal/fs_tool/java_fs_test.go | 18 +++++++-------- .../internal/fs_tool/lc_constructor.go | 10 ++++----- .../run_builder/run_builder_setuper.go | 6 ++--- .../run_builder/run_builder_setuper_test.go | 6 ++--- 12 files changed, 55 insertions(+), 55 deletions(-) diff --git a/playground/backend/cmd/server/controller.go b/playground/backend/cmd/server/controller.go index 5f63e6b56558..6c90474ee981 100644 --- a/playground/backend/cmd/server/controller.go +++ b/playground/backend/cmd/server/controller.go @@ -65,7 +65,7 @@ func (controller *playgroundController) RunCode(ctx context.Context, info *pb.Ru return nil, errors.InternalError("Run code", "Error during setup file system: "+err.Error()) } - compileBuilder, err := compile_builder.Setup(lc.GetAbsoluteSourceFilePath(), lc.GetAbsoluteExecutableFilesFolderPath(), info.Sdk, controller.env.BeamSdkEnvs.ExecutorConfig) + compileBuilder, err := compile_builder.Setup(lc.GetAbsoluteSourceFilePath(), lc.GetAbsoluteBaseFolderPath(), info.Sdk, controller.env.BeamSdkEnvs.ExecutorConfig) if err != nil { logger.Errorf("RunCode(): error during setup run builder: %s\n", err.Error()) return nil, errors.InvalidArgumentError("Run code", "Error during setup compile builder: "+err.Error()) diff --git a/playground/backend/internal/code_processing/code_processing_test.go b/playground/backend/internal/code_processing/code_processing_test.go index 5f0fb2ba8af2..9cb73c9d5404 100644 --- a/playground/backend/internal/code_processing/code_processing_test.go +++ b/playground/backend/internal/code_processing/code_processing_test.go @@ -223,7 +223,7 @@ func Test_Process(t *testing.T) { t.Run(tt.name, func(t *testing.T) { lc, _ := fs_tool.NewLifeCycle(pb.Sdk_SDK_JAVA, tt.args.pipelineId, os.Getenv("APP_WORK_DIR")) filePath := lc.GetAbsoluteSourceFilePath() - workingDir := lc.GetAbsoluteExecutableFilesFolderPath() + workingDir := lc.GetAbsoluteBaseFolderPath() exec := executors.NewExecutorBuilder(). WithValidator(). diff --git a/playground/backend/internal/executors/executor_builder.go b/playground/backend/internal/executors/executor_builder.go index 31f126cc567a..82a9f2e23410 100644 --- a/playground/backend/internal/executors/executor_builder.go +++ b/playground/backend/internal/executors/executor_builder.go @@ -120,8 +120,8 @@ func (b *RunBuilder) WithArgs(runArgs []string) *RunBuilder { return b } -//WithClassName adds file name to executor -func (b *RunBuilder) WithClassName(name string) *RunBuilder { +//WithExecutableName adds file name to executor +func (b *RunBuilder) WithExecutableName(name string) *RunBuilder { b.actions = append(b.actions, func(e *Executor) { e.runArgs.fileName = name }) diff --git a/playground/backend/internal/executors/executor_test.go b/playground/backend/internal/executors/executor_test.go index 977970bfc5b8..fbcfb525e6d5 100644 --- a/playground/backend/internal/executors/executor_test.go +++ b/playground/backend/internal/executors/executor_test.go @@ -51,7 +51,7 @@ func BaseExecutorBuilder(envs environment.BeamEnvs, workingDir string, filePath WithRunner(). WithCommand(envs.ExecutorConfig.RunCmd). WithArgs(envs.ExecutorConfig.RunArgs). - WithClassName("HelloWorld"). + WithExecutableName("HelloWorld"). WithWorkingDir(workingDir). WithValidator(). WithSdkValidators(validatorsFuncs). diff --git a/playground/backend/internal/fs_tool/fs.go b/playground/backend/internal/fs_tool/fs.go index 5e7ceaa22295..5d4721a67518 100644 --- a/playground/backend/internal/fs_tool/fs.go +++ b/playground/backend/internal/fs_tool/fs.go @@ -31,16 +31,16 @@ const ( // Folder contains names of folders with executable and compiled files. // For each SDK these values should be set depending on folders that need for the SDK. type Folder struct { - BaseFolder string - SourceFileFolder string - CompiledFolder string + BaseFolder string + SourceFileFolder string + ExecutableFileFolder string } // Extension contains executable and compiled files' extensions. // For each SDK these values should be set depending on SDK's extensions. type Extension struct { - SourceFileExtension string - CompiledExtension string + SourceFileExtension string + ExecutableFileExtension string } // LifeCycle is used for preparing folders and files to process code for one request. @@ -111,16 +111,16 @@ func (l *LifeCycle) GetAbsoluteSourceFilePath() string { return absoluteFilePath } -// GetAbsoluteBinaryFilePath returns absolute filepath to compiled file (/path/to/workingDir/executable_files/{pipelineId}/bin/{pipelineId}.{compiledExtension}). -func (l *LifeCycle) GetAbsoluteBinaryFilePath() string { - fileName := l.pipelineId.String() + l.Extension.CompiledExtension - filePath := filepath.Join(l.Folder.CompiledFolder, fileName) +// GetAbsoluteExecutableFilePath returns absolute filepath to compiled file (/path/to/workingDir/executable_files/{pipelineId}/bin/{pipelineId}.{executableExtension}). +func (l *LifeCycle) GetAbsoluteExecutableFilePath() string { + fileName := l.pipelineId.String() + l.Extension.ExecutableFileExtension + filePath := filepath.Join(l.Folder.ExecutableFileFolder, fileName) absoluteFilePath, _ := filepath.Abs(filePath) return absoluteFilePath } -// GetAbsoluteExecutableFilesFolderPath returns absolute path to executable folder (/path/to/workingDir/executable_files/{pipelineId}). -func (l *LifeCycle) GetAbsoluteExecutableFilesFolderPath() string { +// GetAbsoluteBaseFolderPath returns absolute path to executable folder (/path/to/workingDir/executable_files/{pipelineId}). +func (l *LifeCycle) GetAbsoluteBaseFolderPath() string { absoluteFilePath, _ := filepath.Abs(l.Folder.BaseFolder) return absoluteFilePath } diff --git a/playground/backend/internal/fs_tool/fs_test.go b/playground/backend/internal/fs_tool/fs_test.go index f87664fa9090..ef3b0a6f19c4 100644 --- a/playground/backend/internal/fs_tool/fs_test.go +++ b/playground/backend/internal/fs_tool/fs_test.go @@ -53,8 +53,8 @@ func TestLifeCycle_CreateExecutableFile(t *testing.T) { name: "executable folder doesn't exist", fields: fields{ folder: Folder{ - SourceFileFolder: srcFileFolder, - CompiledFolder: binFileFolder, + SourceFileFolder: srcFileFolder, + ExecutableFileFolder: binFileFolder, }, pipelineId: pipelineId, }, @@ -208,13 +208,13 @@ func TestNewLifeCycle(t *testing.T) { want: &LifeCycle{ folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ - BaseFolder: baseFileFolder, - SourceFileFolder: srcFileFolder, - CompiledFolder: binFileFolder, + BaseFolder: baseFileFolder, + SourceFileFolder: srcFileFolder, + ExecutableFileFolder: binFileFolder, }, Extension: Extension{ - SourceFileExtension: javaSourceFileExtension, - CompiledExtension: javaCompiledFileExtension, + SourceFileExtension: javaSourceFileExtension, + ExecutableFileExtension: javaCompiledFileExtension, }, ExecutableName: executableName, pipelineId: pipelineId, @@ -338,9 +338,9 @@ func TestLifeCycle_GetAbsoluteExecutableFilesFolderPath(t *testing.T) { Extension: tt.fields.Extension, pipelineId: tt.fields.pipelineId, } - got := l.GetAbsoluteExecutableFilesFolderPath() + got := l.GetAbsoluteBaseFolderPath() if got != tt.want { - t.Errorf("GetAbsoluteExecutableFilesFolderPath() got = %v, want %v", got, tt.want) + t.Errorf("GetAbsoluteBaseFolderPath() got = %v, want %v", got, tt.want) } }) } @@ -369,8 +369,8 @@ func TestLifeCycle_ExecutableName(t *testing.T) { name: "ExecutableName", fields: fields{ Folder: Folder{ - BaseFolder: baseFileFolder, - CompiledFolder: binFileFolder, + BaseFolder: baseFileFolder, + ExecutableFileFolder: binFileFolder, }, ExecutableName: func(u uuid.UUID, s string) (string, error) { return "MOCK_EXECUTABLE_NAME", nil diff --git a/playground/backend/internal/fs_tool/go_fs.go b/playground/backend/internal/fs_tool/go_fs.go index 597211f05023..1c642652fdea 100644 --- a/playground/backend/internal/fs_tool/go_fs.go +++ b/playground/backend/internal/fs_tool/go_fs.go @@ -20,11 +20,11 @@ import ( ) const ( - goExecutableFileExtension = ".go" - goCompiledFileExtension = "" + goSourceFileExtension = ".go" + goExecutableFileExtension = "" ) // newGoLifeCycle creates LifeCycle with go SDK environment. func newGoLifeCycle(pipelineId uuid.UUID, workingDir string) *LifeCycle { - return newCompilingLifeCycle(pipelineId, workingDir, goExecutableFileExtension, goCompiledFileExtension) + return newCompilingLifeCycle(pipelineId, workingDir, goSourceFileExtension, goExecutableFileExtension) } diff --git a/playground/backend/internal/fs_tool/go_fs_test.go b/playground/backend/internal/fs_tool/go_fs_test.go index 433d0b7aafbf..e026d1d9e199 100644 --- a/playground/backend/internal/fs_tool/go_fs_test.go +++ b/playground/backend/internal/fs_tool/go_fs_test.go @@ -49,13 +49,13 @@ func Test_newGoLifeCycle(t *testing.T) { want: &LifeCycle{ folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ - BaseFolder: baseFileFolder, - SourceFileFolder: srcFileFolder, - CompiledFolder: binFileFolder, + BaseFolder: baseFileFolder, + SourceFileFolder: srcFileFolder, + ExecutableFileFolder: binFileFolder, }, Extension: Extension{ - SourceFileExtension: goExecutableFileExtension, - CompiledExtension: goCompiledFileExtension, + SourceFileExtension: goSourceFileExtension, + ExecutableFileExtension: goExecutableFileExtension, }, pipelineId: pipelineId, }, diff --git a/playground/backend/internal/fs_tool/java_fs_test.go b/playground/backend/internal/fs_tool/java_fs_test.go index 175ea5fd19cf..a8baf8f9d5c0 100644 --- a/playground/backend/internal/fs_tool/java_fs_test.go +++ b/playground/backend/internal/fs_tool/java_fs_test.go @@ -51,13 +51,13 @@ func Test_newJavaLifeCycle(t *testing.T) { want: &LifeCycle{ folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ - BaseFolder: baseFileFolder, - SourceFileFolder: srcFileFolder, - CompiledFolder: binFileFolder, + BaseFolder: baseFileFolder, + SourceFileFolder: srcFileFolder, + ExecutableFileFolder: binFileFolder, }, Extension: Extension{ - SourceFileExtension: javaSourceFileExtension, - CompiledExtension: javaCompiledFileExtension, + SourceFileExtension: javaSourceFileExtension, + ExecutableFileExtension: javaCompiledFileExtension, }, ExecutableName: executableName, pipelineId: pipelineId, @@ -103,7 +103,7 @@ func Test_executableName(t *testing.T) { wantErr bool }{ { - // Test case with calling executableName method with correct pipelineId and workingDir. + // Test case with calling sourceFileName method with correct pipelineId and workingDir. // As a result, want to receive a name that should be executed name: "get executable name", prepare: func() { @@ -122,7 +122,7 @@ func Test_executableName(t *testing.T) { wantErr: false, }, { - // Test case with calling executableName method with correct pipelineId and workingDir. + // Test case with calling sourceFileName method with correct pipelineId and workingDir. // As a result, want to receive an error. name: "directory doesn't exist", prepare: func() {}, @@ -139,11 +139,11 @@ func Test_executableName(t *testing.T) { tt.prepare() got, err := executableName(tt.args.pipelineId, tt.args.workingDir) if (err != nil) != tt.wantErr { - t.Errorf("executableName() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("sourceFileName() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("executableName() got = %v, want %v", got, tt.want) + t.Errorf("sourceFileName() got = %v, want %v", got, tt.want) } }) } diff --git a/playground/backend/internal/fs_tool/lc_constructor.go b/playground/backend/internal/fs_tool/lc_constructor.go index dbadbca9beea..a979f825a68e 100644 --- a/playground/backend/internal/fs_tool/lc_constructor.go +++ b/playground/backend/internal/fs_tool/lc_constructor.go @@ -34,13 +34,13 @@ func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, sourceFileEx return &LifeCycle{ folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ - BaseFolder: baseFileFolder, - SourceFileFolder: srcFileFolder, - CompiledFolder: binFileFolder, + BaseFolder: baseFileFolder, + SourceFileFolder: srcFileFolder, + ExecutableFileFolder: binFileFolder, }, Extension: Extension{ - SourceFileExtension: sourceFileExtension, - CompiledExtension: compiledFileExtension, + SourceFileExtension: sourceFileExtension, + ExecutableFileExtension: compiledFileExtension, }, pipelineId: pipelineId, } diff --git a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go index 09bea69437af..e3cc0e7c4903 100644 --- a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go +++ b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go @@ -31,7 +31,7 @@ func Setup(pipelineId uuid.UUID, lc *fs_tool.LifeCycle, workingDir string, sdkEn WithRunner(). WithCommand(sdkEnv.ExecutorConfig.RunCmd). WithArgs(sdkEnv.ExecutorConfig.RunArgs). - WithWorkingDir(lc.GetAbsoluteExecutableFilesFolderPath()) + WithWorkingDir(lc.GetAbsoluteBaseFolderPath()) switch sdkEnv.ApacheBeamSdk { case pb.Sdk_SDK_JAVA: @@ -42,9 +42,9 @@ func Setup(pipelineId uuid.UUID, lc *fs_tool.LifeCycle, workingDir string, sdkEn } runBuilder = runBuilder. - WithClassName(className) + WithExecutableName(className) case pb.Sdk_SDK_GO: - runBuilder = runBuilder.WithCommand(lc.GetAbsoluteBinaryFilePath()) + runBuilder = runBuilder.WithCommand(lc.GetAbsoluteExecutableFilePath()) default: return nil, fmt.Errorf("incorrect sdk: %s", sdkEnv.ApacheBeamSdk) } diff --git a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go index f1264847abe5..c07f7dc98dec 100644 --- a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go +++ b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go @@ -43,7 +43,7 @@ func TestSetup(t *testing.T) { if err != nil { panic(err) } - _, err = os.Create(successLc.Folder.CompiledFolder + "/temp.class") + _, err = os.Create(successLc.Folder.ExecutableFileFolder + "/temp.class") if err != nil { panic(err) } @@ -116,8 +116,8 @@ func TestSetup(t *testing.T) { WithRunner(). WithCommand(sdkEnv.ExecutorConfig.RunCmd). WithArgs(sdkEnv.ExecutorConfig.RunArgs). - WithWorkingDir(successLc.GetAbsoluteExecutableFilesFolderPath()). - WithClassName(className), + WithWorkingDir(successLc.GetAbsoluteBaseFolderPath()). + WithExecutableName(className), wantErr: false, }, } From 2957ff2fafccbd11cdd695d8352d379565b9d609 Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Mon, 22 Nov 2021 13:49:34 +0300 Subject: [PATCH 06/12] working python sdk --- playground/backend/configs/SDK_PYTHON.json | 2 +- .../backend/internal/fs_tool/lc_constructor.go | 14 ++++++++------ .../backend/internal/fs_tool/python_fs_test.go | 7 +++++-- .../setup_tools/run_builder/run_builder_setuper.go | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/playground/backend/configs/SDK_PYTHON.json b/playground/backend/configs/SDK_PYTHON.json index ddb781d6f354..9c2c0858600a 100644 --- a/playground/backend/configs/SDK_PYTHON.json +++ b/playground/backend/configs/SDK_PYTHON.json @@ -1,6 +1,6 @@ { "compile_cmd": "", - "run_cmd": "python", + "run_cmd": "python3", "compile_args": [], "run_args": [] } \ No newline at end of file diff --git a/playground/backend/internal/fs_tool/lc_constructor.go b/playground/backend/internal/fs_tool/lc_constructor.go index 29277184f109..d2a6164a141b 100644 --- a/playground/backend/internal/fs_tool/lc_constructor.go +++ b/playground/backend/internal/fs_tool/lc_constructor.go @@ -47,16 +47,18 @@ func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, sourceFileEx } // newInterpretedLifeCycle creates LifeCycle for interpreted SDK environment. -func newInterpretedLifeCycle(pipelineId uuid.UUID, workingDir string, executableFileExtension string) *LifeCycle { - executableFileFolder := filepath.Join(workingDir, baseFileFolder, pipelineId.String()) +func newInterpretedLifeCycle(pipelineId uuid.UUID, workingDir string, sourceFileExtension string) *LifeCycle { + sourceFileFolder := filepath.Join(workingDir, baseFileFolder, pipelineId.String()) return &LifeCycle{ - folderGlobs: []string{executableFileFolder}, + folderGlobs: []string{sourceFileFolder}, Folder: Folder{ - BaseFolder: executableFileFolder, - ExecutableFolder: executableFileFolder, + BaseFolder: sourceFileFolder, + SourceFileFolder: sourceFileFolder, + ExecutableFileFolder: sourceFileFolder, }, Extension: Extension{ - ExecutableExtension: executableFileExtension, + ExecutableFileExtension: sourceFileExtension, + SourceFileExtension: sourceFileExtension, }, pipelineId: pipelineId, } diff --git a/playground/backend/internal/fs_tool/python_fs_test.go b/playground/backend/internal/fs_tool/python_fs_test.go index f3f0515fc940..a6c848ce17ae 100644 --- a/playground/backend/internal/fs_tool/python_fs_test.go +++ b/playground/backend/internal/fs_tool/python_fs_test.go @@ -47,10 +47,13 @@ func Test_newPythonLifeCycle(t *testing.T) { want: &LifeCycle{ folderGlobs: []string{baseFileFolder}, Folder: Folder{ - BaseFolder: baseFileFolder, + BaseFolder: baseFileFolder, + SourceFileFolder: baseFileFolder, + ExecutableFileFolder: baseFileFolder, }, Extension: Extension{ - ExecutableExtension: pythonExecutableFileExtension, + SourceFileExtension: pythonExecutableFileExtension, + ExecutableFileExtension: pythonExecutableFileExtension, }, pipelineId: pipelineId, }, diff --git a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go index fdf420bf51e3..32199cb2724d 100644 --- a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go +++ b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go @@ -46,7 +46,7 @@ func Setup(pipelineId uuid.UUID, lc *fs_tool.LifeCycle, workingDir string, sdkEn case pb.Sdk_SDK_GO: runBuilder = runBuilder.WithCommand(lc.GetAbsoluteExecutableFilePath()) case pb.Sdk_SDK_PYTHON: - runBuilder = runBuilder.WithCommand(lc.GetAbsoluteExecutableFilePath()) + runBuilder = runBuilder.WithExecutableName(lc.GetAbsoluteExecutableFilePath()) default: return nil, fmt.Errorf("incorrect sdk: %s", sdkEnv.ApacheBeamSdk) } From f240b105326fadf96cd486671e1120311ba4e982 Mon Sep 17 00:00:00 2001 From: "daria.malkova" Date: Mon, 22 Nov 2021 18:29:59 +0300 Subject: [PATCH 07/12] Update playground/backend/internal/fs_tool/python_fs_test.go Co-authored-by: Aydar Zainutdinov --- playground/backend/internal/fs_tool/python_fs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/backend/internal/fs_tool/python_fs_test.go b/playground/backend/internal/fs_tool/python_fs_test.go index a6c848ce17ae..b2e41e88f0a5 100644 --- a/playground/backend/internal/fs_tool/python_fs_test.go +++ b/playground/backend/internal/fs_tool/python_fs_test.go @@ -37,7 +37,7 @@ func Test_newPythonLifeCycle(t *testing.T) { want *LifeCycle }{ { - // Test case with calling newGoLifeCycle method with correct pipelineId and workingDir. + // Test case with calling newPythonLifeCycle method with correct pipelineId and workingDir. // As a result, want to receive an expected go life cycle. name: "newPythonLifeCycle", args: args{ From 05878a71c4b7145664967379079753b7f0cc8ccf Mon Sep 17 00:00:00 2001 From: "daria.malkova" Date: Mon, 22 Nov 2021 18:30:05 +0300 Subject: [PATCH 08/12] Update playground/backend/internal/fs_tool/python_fs_test.go Co-authored-by: Aydar Zainutdinov --- playground/backend/internal/fs_tool/python_fs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/backend/internal/fs_tool/python_fs_test.go b/playground/backend/internal/fs_tool/python_fs_test.go index b2e41e88f0a5..c0e6ded2755c 100644 --- a/playground/backend/internal/fs_tool/python_fs_test.go +++ b/playground/backend/internal/fs_tool/python_fs_test.go @@ -38,7 +38,7 @@ func Test_newPythonLifeCycle(t *testing.T) { }{ { // Test case with calling newPythonLifeCycle method with correct pipelineId and workingDir. - // As a result, want to receive an expected go life cycle. + // As a result, want to receive an expected python life cycle. name: "newPythonLifeCycle", args: args{ pipelineId: pipelineId, From 02412f84b00701d16bc841ae77828393efbde6bc Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Tue, 23 Nov 2021 12:49:54 +0300 Subject: [PATCH 09/12] refactoring + adding dockerfile --- playground/backend/configs/SDK_PYTHON.json | 2 +- .../backend/containers/python/Dockerfile | 51 ++++++++++++++ playground/backend/containers/python/app.yaml | 22 ++++++ .../backend/containers/python/build.gradle | 69 +++++++++++++++++++ .../backend/containers/python/settings.gradle | 19 +++++ .../code_processing/code_processing.go | 23 +++---- settings.gradle.kts | 1 + 7 files changed, 174 insertions(+), 13 deletions(-) create mode 100644 playground/backend/containers/python/Dockerfile create mode 100644 playground/backend/containers/python/app.yaml create mode 100644 playground/backend/containers/python/build.gradle create mode 100644 playground/backend/containers/python/settings.gradle diff --git a/playground/backend/configs/SDK_PYTHON.json b/playground/backend/configs/SDK_PYTHON.json index 9c2c0858600a..bb6ae89adff5 100644 --- a/playground/backend/configs/SDK_PYTHON.json +++ b/playground/backend/configs/SDK_PYTHON.json @@ -3,4 +3,4 @@ "run_cmd": "python3", "compile_args": [], "run_args": [] -} \ No newline at end of file +} diff --git a/playground/backend/containers/python/Dockerfile b/playground/backend/containers/python/Dockerfile new file mode 100644 index 000000000000..d02dff63e1d2 --- /dev/null +++ b/playground/backend/containers/python/Dockerfile @@ -0,0 +1,51 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +ARG BASE_IMAGE=apache/beam_python3.7_sdk:latest +ARG GO_BASE_IMAGE=golang:1.17-buster +FROM $GO_BASE_IMAGE AS build + +# Setup Go Environment +ENV GOPATH /go +ENV PATH $GOPATH/bin:$PATH +RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1 &&\ + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 + +# Prepare Application +COPY src /go/src/playground/backend +#COPY playground /go/src/playground/playground +WORKDIR /go/src/playground/backend +RUN ls +# Build Application +RUN go mod download &&\ + go mod tidy &&\ + cd cmd/server &&\ + go build -o /go/bin/server_python_backend + +FROM $BASE_IMAGE +ENV SERVER_IP=0.0.0.0 +ENV SERVER_PORT=8080 +ENV APP_WORK_DIR=/opt/playground/backend/ +ENV BEAM_SDK="SDK_PYTHON" + +# Copy build result +COPY --from=build /go/bin/server_python_backend /opt/playground/backend/ +COPY --from=build /go/src/playground/backend/configs /opt/playground/backend/configs/ + +RUN printenv +ENTRYPOINT ["/opt/playground/backend/server_python_backend"] diff --git a/playground/backend/containers/python/app.yaml b/playground/backend/containers/python/app.yaml new file mode 100644 index 000000000000..28fb02e84195 --- /dev/null +++ b/playground/backend/containers/python/app.yaml @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +service: backend-python +runtime: custom +env: flex +manual_scaling: + instances: 1 diff --git a/playground/backend/containers/python/build.gradle b/playground/backend/containers/python/build.gradle new file mode 100644 index 000000000000..5e2e20b38ec7 --- /dev/null +++ b/playground/backend/containers/python/build.gradle @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +apply plugin: 'org.apache.beam.module' +apply plugin: 'base' +applyDockerNature() + +def playgroundJobServerProject = "${project.path.replace('-container', '')}" + +description = project(playgroundJobServerProject).description + " :: Container" + +configurations { + dockerDependency +} + +dependencies { + dockerDependency project(path: playgroundJobServerProject, configuration: "shadow") +} + +task copyDockerfileDependencies(type: Copy) { + copy { + from '../../../backend/' + into 'build/src' + exclude 'containers' + } + copy { + from 'Dockerfile' + into 'build/' + } + copy { + from '../../../playground' + into 'build/playground' + } +} + +docker { + name containerImageName( + name: project.docker_image_default_repo_prefix + "playground-backend-python", + root: project.rootProject.hasProperty(["docker-repository-root"]) ? + project.rootProject["docker-repository-root"] : + project.docker_image_default_repo_root) + files "./build/" + tags containerImageTags() + buildArgs(['BASE_IMAGE': project.rootProject.hasProperty(["base-image"]) ? + project.rootProject["base-image"] : + "apache/beam_python3.7_sdk" ]) + buildArgs(['GO_BASE_IMAGE': project.rootProject.hasProperty(["go-base-image"]) ? + project.rootProject["go-base-image"] : + "golang:1.17-buster" ]) +} + +// Ensure that we build the required resources and copy and file dependencies from related projects +dockerPrepare.dependsOn copyDockerfileDependencies diff --git a/playground/backend/containers/python/settings.gradle b/playground/backend/containers/python/settings.gradle new file mode 100644 index 000000000000..63b23b446ded --- /dev/null +++ b/playground/backend/containers/python/settings.gradle @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +rootProject.name = 'apache-beam-playground-backend-python' diff --git a/playground/backend/internal/code_processing/code_processing.go b/playground/backend/internal/code_processing/code_processing.go index 90bcd9fe5ee5..bd2430428d48 100644 --- a/playground/backend/internal/code_processing/code_processing.go +++ b/playground/backend/internal/code_processing/code_processing.go @@ -79,22 +79,21 @@ func Process(ctx context.Context, cacheService cache.Cache, lc *fs_tool.LifeCycl } switch sdkEnv.ApacheBeamSdk { - case pb.Sdk_SDK_JAVA: - case pb.Sdk_SDK_GO: + case pb.Sdk_SDK_JAVA, pb.Sdk_SDK_GO: // compile logger.Infof("%s: Compile() ...\n", pipelineId) compileCmd := executor.Compile(ctxWithTimeout) go func(cmd *exec.Cmd, successChannel chan bool, errChannel chan error, dataChannel chan interface{}) { - // TODO separate stderr from stdout [BEAM-13208] - data, err := cmd.CombinedOutput() - dataChannel <- data - if err != nil { - errChannel <- err - successChannel <- false - } else { - successChannel <- true - } - }(compileCmd, successChannel, errorChannel, dataChannel) + // TODO separate stderr from stdout [BEAM-13208] + data, err := cmd.CombinedOutput() + dataChannel <- data + if err != nil { + errChannel <- err + successChannel <- false + } else { + successChannel <- true + } + }(compileCmd, successChannel, errorChannel, dataChannel) if err := processStep(ctxWithTimeout, pipelineId, cacheService, cancelChannel, successChannel, dataChannel, nil, errorChannel, pb.Status_STATUS_COMPILE_ERROR, pb.Status_STATUS_EXECUTING); err != nil { return diff --git a/settings.gradle.kts b/settings.gradle.kts index 9e3d5bae3d73..0b91a663520a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -60,6 +60,7 @@ include(":playground") include(":playground:backend") include(":playground:frontend") include(":playground:backend:containers:java") +include(":playground:backend:containers:python") include(":runners:core-construction-java") include(":runners:core-java") include(":runners:direct-java") From 816d46c733afdb3b7a5fb51c2dc3088167f99449 Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Tue, 23 Nov 2021 18:28:31 +0300 Subject: [PATCH 10/12] add preparators array (will be filled later) --- .../preparators/python_preparators.go | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 playground/backend/internal/preparators/python_preparators.go diff --git a/playground/backend/internal/preparators/python_preparators.go b/playground/backend/internal/preparators/python_preparators.go new file mode 100644 index 000000000000..d2ff96382bfb --- /dev/null +++ b/playground/backend/internal/preparators/python_preparators.go @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package preparators + +const () + +// GetPythonPreparators returns preparation methods that should be applied to Python code +func GetPythonPreparators(filePath string) *[]Preparator { + return &[]Preparator{} +} From 2b7569c04a391ad5862b836662065cf0cf31c346 Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Wed, 24 Nov 2021 12:39:47 +0300 Subject: [PATCH 11/12] fix indentation --- .../backend/internal/code_processing/code_processing.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playground/backend/internal/code_processing/code_processing.go b/playground/backend/internal/code_processing/code_processing.go index ccd55ab41248..948b88f8e86a 100644 --- a/playground/backend/internal/code_processing/code_processing.go +++ b/playground/backend/internal/code_processing/code_processing.go @@ -84,8 +84,8 @@ func Process(ctx context.Context, cacheService cache.Cache, lc *fs_tool.LifeCycl logger.Infof("%s: Compile() ...\n", pipelineId) compileCmd := executor.Compile(ctxWithTimeout) var compileError bytes.Buffer - var compileOutput bytes.Buffer - runCmdWithOutput(compileCmd, &compileOutput, &compileError, successChannel, errorChannel) + var compileOutput bytes.Buffer + runCmdWithOutput(compileCmd, &compileOutput, &compileError, successChannel, errorChannel) if err := processStep(ctxWithTimeout, pipelineId, cacheService, cancelChannel, successChannel, &compileOutput, &compileError, errorChannel, pb.Status_STATUS_COMPILE_ERROR, pb.Status_STATUS_EXECUTING); err != nil { return From 42a47e9368471de1da556193629a591736c365a9 Mon Sep 17 00:00:00 2001 From: AydarZaynutdinov Date: Wed, 24 Nov 2021 19:36:13 +0300 Subject: [PATCH 12/12] [BEAM-13307][Playground] Fix tests; Update processing of the compile step for python sdk; --- .../backend/internal/code_processing/code_processing.go | 2 ++ playground/backend/internal/executors/executor_test.go | 2 +- .../setup_tools/run_builder/run_builder_setuper_test.go | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/playground/backend/internal/code_processing/code_processing.go b/playground/backend/internal/code_processing/code_processing.go index 948b88f8e86a..8d81d35f97e6 100644 --- a/playground/backend/internal/code_processing/code_processing.go +++ b/playground/backend/internal/code_processing/code_processing.go @@ -90,6 +90,8 @@ func Process(ctx context.Context, cacheService cache.Cache, lc *fs_tool.LifeCycl if err := processStep(ctxWithTimeout, pipelineId, cacheService, cancelChannel, successChannel, &compileOutput, &compileError, errorChannel, pb.Status_STATUS_COMPILE_ERROR, pb.Status_STATUS_EXECUTING); err != nil { return } + case pb.Sdk_SDK_PYTHON: + processSuccess(ctx, []byte(""), pipelineId, cacheService, pb.Status_STATUS_EXECUTING) } runBuilder, err := run_builder.Setup(pipelineId, lc, appEnv.WorkingDir(), sdkEnv, compileBuilder) diff --git a/playground/backend/internal/executors/executor_test.go b/playground/backend/internal/executors/executor_test.go index fbcfb525e6d5..08e687398f9a 100644 --- a/playground/backend/internal/executors/executor_test.go +++ b/playground/backend/internal/executors/executor_test.go @@ -29,7 +29,7 @@ import ( var ( executorConfig = environment.NewExecutorConfig("javac", "java", []string{"-d", "bin", "-classpath"}, []string{"-cp", "bin:"}) - env = environment.NewEnvironment(environment.NetworkEnvs{}, *environment.NewBeamEnvs(pb.Sdk_SDK_JAVA, executorConfig), environment.ApplicationEnvs{}) + env = environment.NewEnvironment(environment.NetworkEnvs{}, *environment.NewBeamEnvs(pb.Sdk_SDK_JAVA, executorConfig, ""), environment.ApplicationEnvs{}) ) // BaseExecutorBuilder fills up an executor with base parameters diff --git a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go index c07f7dc98dec..41c05dc37adb 100644 --- a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go +++ b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go @@ -53,7 +53,7 @@ func TestSetup(t *testing.T) { CompileArgs: []string{"MOCK_COMPILE_ARG"}, RunArgs: []string{"MOCK_RUN_ARG"}, } - sdkEnv := environment.NewBeamEnvs(playground.Sdk_SDK_JAVA, executorConfig) + sdkEnv := environment.NewBeamEnvs(playground.Sdk_SDK_JAVA, executorConfig, "") compileBuilder := &executors.CompileBuilder{} className, err := successLc.ExecutableName(successPipelineId, "") if err != nil { @@ -81,7 +81,7 @@ func TestSetup(t *testing.T) { pipelineId: successPipelineId, lc: successLc, workingDir: "", - sdkEnv: environment.NewBeamEnvs(playground.Sdk_SDK_UNSPECIFIED, executorConfig), + sdkEnv: environment.NewBeamEnvs(playground.Sdk_SDK_UNSPECIFIED, executorConfig, ""), compileBuilder: compileBuilder, }, want: nil,