From b73f5f70ac4184b56a0a03922731c5f2f69b9566 Mon Sep 17 00:00:00 2001 From: Sergey Kalinin <91209855+snkalinin@users.noreply.github.com> Date: Tue, 16 Nov 2021 14:59:11 +0300 Subject: [PATCH 1/4] [Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83) * Update workflows for playground * Attempt to fix tests * Remove continue on error to catch errors * Fix linter problem for backend dockerfile * Update folder to run backend go linter * Moved flutter test to execution via gradle tasks --- .../workflows/build_playground_backend.yml | 14 ++++++++----- .../workflows/build_playground_frontend.yml | 20 +++++++------------ playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index 96e8aae9b0d5..bd2868768246 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,10 +17,8 @@ name: Build And Deploy Playground Backend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -45,12 +43,14 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint - run: "golangci-lint run internal/..." +# run: "golangci-lint run internal/..." + run: "golangci-lint run cmd/server/..." working-directory: playground/backend/ - continue-on-error: true + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue + run: rm ~/.m2/settings.xml - name: Run Tests run: ./gradlew playground:backend:test - continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,12 +62,16 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index 9c2d56a52428..f93e330442b0 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,12 +17,10 @@ name: Build And Deploy Playground Frontend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/backend/**'] + paths: ['playground/frontend/**'] workflow_dispatch: jobs: @@ -43,18 +41,10 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' - - name: Prepare Flutter lint - run: "flutter pub add flutter_lints --dev" - working-directory: playground/frontend/ - continue-on-error: true - name: Run Lint - run: "flutter analyze" - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:analyze - name: Run Tests - run: flutter test - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:test - name: install npm uses: actions/setup-node@v2 with: @@ -66,12 +56,16 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index d7275020dcfa..8846368ff8b6 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE +ARG BASE_IMAGE=apache/beam_java8_sdk:latest FROM golang:1.17-buster AS build # Setup Go Environment From d449275b8ebd623760b34981f56c8f62b140ad9d Mon Sep 17 00:00:00 2001 From: Ilya Date: Tue, 16 Nov 2021 15:07:49 +0300 Subject: [PATCH 2/4] Revert "[Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83)" (#88) This reverts commit b73f5f70ac4184b56a0a03922731c5f2f69b9566. --- .../workflows/build_playground_backend.yml | 14 +++++-------- .../workflows/build_playground_frontend.yml | 20 ++++++++++++------- playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index bd2868768246..96e8aae9b0d5 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,8 +17,10 @@ name: Build And Deploy Playground Backend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -43,14 +45,12 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint -# run: "golangci-lint run internal/..." - run: "golangci-lint run cmd/server/..." + run: "golangci-lint run internal/..." working-directory: playground/backend/ - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: rm ~/.m2/settings.xml + continue-on-error: true - name: Run Tests run: ./gradlew playground:backend:test + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,16 +62,12 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index f93e330442b0..9c2d56a52428 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,10 +17,12 @@ name: Build And Deploy Playground Frontend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/frontend/**'] + paths: ['playground/backend/**'] workflow_dispatch: jobs: @@ -41,10 +43,18 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' + - name: Prepare Flutter lint + run: "flutter pub add flutter_lints --dev" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Lint - run: ./gradlew playground:frontend:analyze + run: "flutter analyze" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Tests - run: ./gradlew playground:frontend:test + run: flutter test + working-directory: playground/frontend/ + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -56,16 +66,12 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index 8846368ff8b6..d7275020dcfa 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE=apache/beam_java8_sdk:latest +ARG BASE_IMAGE FROM golang:1.17-buster AS build # Setup Go Environment From df5e0530db506305702d8a3b812fa162e64e4c09 Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Mon, 22 Nov 2021 11:15:39 +0300 Subject: [PATCH 3/4] Rename LF executable -> source --- playground/backend/cmd/server/controller.go | 2 +- .../code_processing/code_processing_test.go | 6 ++-- playground/backend/internal/fs_tool/fs.go | 22 ++++++------- .../backend/internal/fs_tool/fs_test.go | 32 +++++++++---------- .../backend/internal/fs_tool/go_fs_test.go | 4 +-- .../backend/internal/fs_tool/java_fs.go | 6 ++-- .../backend/internal/fs_tool/java_fs_test.go | 4 +-- .../{compiling_lc.go => lc_constructor.go} | 8 ++--- .../preparators/java_preparators_test.go | 4 +-- .../life_cycle/life_cycle_setuper.go | 4 +-- 10 files changed, 46 insertions(+), 46 deletions(-) rename playground/backend/internal/fs_tool/{compiling_lc.go => lc_constructor.go} (86%) diff --git a/playground/backend/cmd/server/controller.go b/playground/backend/cmd/server/controller.go index 61b581a7ca6b..5f63e6b56558 100644 --- a/playground/backend/cmd/server/controller.go +++ b/playground/backend/cmd/server/controller.go @@ -65,7 +65,7 @@ func (controller *playgroundController) RunCode(ctx context.Context, info *pb.Ru return nil, errors.InternalError("Run code", "Error during setup file system: "+err.Error()) } - compileBuilder, err := compile_builder.Setup(lc.GetAbsoluteExecutableFilePath(), lc.GetAbsoluteExecutableFilesFolderPath(), info.Sdk, controller.env.BeamSdkEnvs.ExecutorConfig) + compileBuilder, err := compile_builder.Setup(lc.GetAbsoluteSourceFilePath(), lc.GetAbsoluteExecutableFilesFolderPath(), info.Sdk, controller.env.BeamSdkEnvs.ExecutorConfig) if err != nil { logger.Errorf("RunCode(): error during setup run builder: %s\n", err.Error()) return nil, errors.InvalidArgumentError("Run code", "Error during setup compile builder: "+err.Error()) diff --git a/playground/backend/internal/code_processing/code_processing_test.go b/playground/backend/internal/code_processing/code_processing_test.go index 1fe0ea312d94..5f0fb2ba8af2 100644 --- a/playground/backend/internal/code_processing/code_processing_test.go +++ b/playground/backend/internal/code_processing/code_processing_test.go @@ -222,7 +222,7 @@ func Test_Process(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { lc, _ := fs_tool.NewLifeCycle(pb.Sdk_SDK_JAVA, tt.args.pipelineId, os.Getenv("APP_WORK_DIR")) - filePath := lc.GetAbsoluteExecutableFilePath() + filePath := lc.GetAbsoluteSourceFilePath() workingDir := lc.GetAbsoluteExecutableFilesFolderPath() exec := executors.NewExecutorBuilder(). @@ -239,7 +239,7 @@ func Test_Process(t *testing.T) { t.Fatalf("error during prepare folders: %s", err.Error()) } if tt.createExecFile { - _, _ = lc.CreateExecutableFile(tt.code) + _, _ = lc.CreateSourceCodeFile(tt.code) } if tt.cancelFunc { @@ -258,7 +258,7 @@ func Test_Process(t *testing.T) { compileOutput, _ := cacheService.GetValue(tt.args.ctx, tt.args.pipelineId, cache.CompileOutput) if tt.expectedCompileOutput != nil && strings.Contains(tt.expectedCompileOutput.(string), "%s") { - tt.expectedCompileOutput = fmt.Sprintf(tt.expectedCompileOutput.(string), lc.GetAbsoluteExecutableFilePath()) + tt.expectedCompileOutput = fmt.Sprintf(tt.expectedCompileOutput.(string), lc.GetAbsoluteSourceFilePath()) } if !reflect.DeepEqual(compileOutput, tt.expectedCompileOutput) { t.Errorf("processCode() set compileOutput: %s, but expectes: %s", compileOutput, tt.expectedCompileOutput) diff --git a/playground/backend/internal/fs_tool/fs.go b/playground/backend/internal/fs_tool/fs.go index 77ba7e72ef0a..5e7ceaa22295 100644 --- a/playground/backend/internal/fs_tool/fs.go +++ b/playground/backend/internal/fs_tool/fs.go @@ -32,14 +32,14 @@ const ( // For each SDK these values should be set depending on folders that need for the SDK. type Folder struct { BaseFolder string - ExecutableFolder string + SourceFileFolder string CompiledFolder string } // Extension contains executable and compiled files' extensions. // For each SDK these values should be set depending on SDK's extensions. type Extension struct { - ExecutableExtension string + SourceFileExtension string CompiledExtension string } @@ -88,14 +88,14 @@ func (l *LifeCycle) DeleteFolders() error { return nil } -// CreateExecutableFile creates an executable file (i.e. file.{executableExtension}). -func (l *LifeCycle) CreateExecutableFile(code string) (string, error) { - if _, err := os.Stat(l.Folder.ExecutableFolder); os.IsNotExist(err) { +// CreateSourceCodeFile creates an executable file (i.e. file.{sourceFileExtension}). +func (l *LifeCycle) CreateSourceCodeFile(code string) (string, error) { + if _, err := os.Stat(l.Folder.SourceFileFolder); os.IsNotExist(err) { return "", err } - fileName := l.pipelineId.String() + l.Extension.ExecutableExtension - filePath := filepath.Join(l.Folder.ExecutableFolder, fileName) + fileName := l.pipelineId.String() + l.Extension.SourceFileExtension + filePath := filepath.Join(l.Folder.SourceFileFolder, fileName) err := os.WriteFile(filePath, []byte(code), fileMode) if err != nil { return "", err @@ -103,10 +103,10 @@ func (l *LifeCycle) CreateExecutableFile(code string) (string, error) { return fileName, nil } -// GetAbsoluteExecutableFilePath returns absolute filepath to executable file (/path/to/workingDir/executable_files/{pipelineId}/src/{pipelineId}.{executableExtension}). -func (l *LifeCycle) GetAbsoluteExecutableFilePath() string { - fileName := l.pipelineId.String() + l.Extension.ExecutableExtension - filePath := filepath.Join(l.Folder.ExecutableFolder, fileName) +// GetAbsoluteSourceFilePath returns absolute filepath to executable file (/path/to/workingDir/executable_files/{pipelineId}/src/{pipelineId}.{sourceFileExtension}). +func (l *LifeCycle) GetAbsoluteSourceFilePath() string { + fileName := l.pipelineId.String() + l.Extension.SourceFileExtension + filePath := filepath.Join(l.Folder.SourceFileFolder, fileName) absoluteFilePath, _ := filepath.Abs(filePath) return absoluteFilePath } diff --git a/playground/backend/internal/fs_tool/fs_test.go b/playground/backend/internal/fs_tool/fs_test.go index 9e9f66c691cf..f87664fa9090 100644 --- a/playground/backend/internal/fs_tool/fs_test.go +++ b/playground/backend/internal/fs_tool/fs_test.go @@ -53,7 +53,7 @@ func TestLifeCycle_CreateExecutableFile(t *testing.T) { name: "executable folder doesn't exist", fields: fields{ folder: Folder{ - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, CompiledFolder: binFileFolder, }, pipelineId: pipelineId, @@ -66,12 +66,12 @@ func TestLifeCycle_CreateExecutableFile(t *testing.T) { name: "executable folder exists", createFolders: []string{srcFileFolder}, fields: fields{ - folder: Folder{ExecutableFolder: srcFileFolder}, - extension: Extension{ExecutableExtension: javaExecutableFileExtension}, + folder: Folder{SourceFileFolder: srcFileFolder}, + extension: Extension{SourceFileExtension: javaSourceFileExtension}, pipelineId: pipelineId, }, args: args{code: "TEST_CODE"}, - want: pipelineId.String() + javaExecutableFileExtension, + want: pipelineId.String() + javaSourceFileExtension, wantErr: false, }, } @@ -86,13 +86,13 @@ func TestLifeCycle_CreateExecutableFile(t *testing.T) { Extension: tt.fields.extension, pipelineId: tt.fields.pipelineId, } - got, err := l.CreateExecutableFile(tt.args.code) + got, err := l.CreateSourceCodeFile(tt.args.code) if (err != nil) != tt.wantErr { - t.Errorf("CreateExecutableFile() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("CreateSourceCodeFile() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("CreateExecutableFile() got = %v, want %v", got, tt.want) + t.Errorf("CreateSourceCodeFile() got = %v, want %v", got, tt.want) } }) os.RemoveAll(baseFileFolder) @@ -209,11 +209,11 @@ func TestNewLifeCycle(t *testing.T) { folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ BaseFolder: baseFileFolder, - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, CompiledFolder: binFileFolder, }, Extension: Extension{ - ExecutableExtension: javaExecutableFileExtension, + SourceFileExtension: javaSourceFileExtension, CompiledExtension: javaCompiledFileExtension, }, ExecutableName: executableName, @@ -260,7 +260,7 @@ func TestLifeCycle_GetAbsoluteExecutableFilePath(t *testing.T) { baseFileFolder := fmt.Sprintf("%s_%s", baseFileFolder, pipelineId) srcFileFolder := baseFileFolder + "/src" - filePath := fmt.Sprintf("%s/%s", srcFileFolder, pipelineId.String()+javaExecutableFileExtension) + filePath := fmt.Sprintf("%s/%s", srcFileFolder, pipelineId.String()+javaSourceFileExtension) absolutePath, _ := filepath.Abs(filePath) type fields struct { folderGlobs []string @@ -275,13 +275,13 @@ func TestLifeCycle_GetAbsoluteExecutableFilePath(t *testing.T) { wantErr bool }{ { - name: "GetAbsoluteExecutableFilePath", + name: "GetAbsoluteSourceFilePath", fields: fields{ Folder: Folder{ BaseFolder: baseFileFolder, - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, }, - Extension: Extension{ExecutableExtension: javaExecutableFileExtension}, + Extension: Extension{SourceFileExtension: javaSourceFileExtension}, pipelineId: pipelineId, }, want: absolutePath, @@ -295,9 +295,9 @@ func TestLifeCycle_GetAbsoluteExecutableFilePath(t *testing.T) { Extension: tt.fields.Extension, pipelineId: tt.fields.pipelineId, } - got := l.GetAbsoluteExecutableFilePath() + got := l.GetAbsoluteSourceFilePath() if got != tt.want { - t.Errorf("GetAbsoluteExecutableFilePath() got = %v, want %v", got, tt.want) + t.Errorf("GetAbsoluteSourceFilePath() got = %v, want %v", got, tt.want) } }) } @@ -324,7 +324,7 @@ func TestLifeCycle_GetAbsoluteExecutableFilesFolderPath(t *testing.T) { name: "GetAbsoluteExecutableFolderPath", fields: fields{ Folder: Folder{BaseFolder: baseFileFolder}, - Extension: Extension{ExecutableExtension: javaExecutableFileExtension}, + Extension: Extension{SourceFileExtension: javaSourceFileExtension}, pipelineId: pipelineId, }, want: absolutePath, diff --git a/playground/backend/internal/fs_tool/go_fs_test.go b/playground/backend/internal/fs_tool/go_fs_test.go index f8925ce5d32f..433d0b7aafbf 100644 --- a/playground/backend/internal/fs_tool/go_fs_test.go +++ b/playground/backend/internal/fs_tool/go_fs_test.go @@ -50,11 +50,11 @@ func Test_newGoLifeCycle(t *testing.T) { folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ BaseFolder: baseFileFolder, - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, CompiledFolder: binFileFolder, }, Extension: Extension{ - ExecutableExtension: goExecutableFileExtension, + SourceFileExtension: goExecutableFileExtension, CompiledExtension: goCompiledFileExtension, }, pipelineId: pipelineId, diff --git a/playground/backend/internal/fs_tool/java_fs.go b/playground/backend/internal/fs_tool/java_fs.go index e89773040526..b2c81ccb7ab8 100644 --- a/playground/backend/internal/fs_tool/java_fs.go +++ b/playground/backend/internal/fs_tool/java_fs.go @@ -24,13 +24,13 @@ import ( ) const ( - javaExecutableFileExtension = ".java" - javaCompiledFileExtension = ".class" + javaSourceFileExtension = ".java" + javaCompiledFileExtension = ".class" ) // newJavaLifeCycle creates LifeCycle with java SDK environment. func newJavaLifeCycle(pipelineId uuid.UUID, workingDir string) *LifeCycle { - javaLifeCycle := newCompilingLifeCycle(pipelineId, workingDir, javaExecutableFileExtension, javaCompiledFileExtension) + javaLifeCycle := newCompilingLifeCycle(pipelineId, workingDir, javaSourceFileExtension, javaCompiledFileExtension) javaLifeCycle.ExecutableName = executableName return javaLifeCycle } diff --git a/playground/backend/internal/fs_tool/java_fs_test.go b/playground/backend/internal/fs_tool/java_fs_test.go index a65e4562cea0..175ea5fd19cf 100644 --- a/playground/backend/internal/fs_tool/java_fs_test.go +++ b/playground/backend/internal/fs_tool/java_fs_test.go @@ -52,11 +52,11 @@ func Test_newJavaLifeCycle(t *testing.T) { folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ BaseFolder: baseFileFolder, - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, CompiledFolder: binFileFolder, }, Extension: Extension{ - ExecutableExtension: javaExecutableFileExtension, + SourceFileExtension: javaSourceFileExtension, CompiledExtension: javaCompiledFileExtension, }, ExecutableName: executableName, diff --git a/playground/backend/internal/fs_tool/compiling_lc.go b/playground/backend/internal/fs_tool/lc_constructor.go similarity index 86% rename from playground/backend/internal/fs_tool/compiling_lc.go rename to playground/backend/internal/fs_tool/lc_constructor.go index 94136eda2dc5..dbadbca9beea 100644 --- a/playground/backend/internal/fs_tool/compiling_lc.go +++ b/playground/backend/internal/fs_tool/lc_constructor.go @@ -26,8 +26,8 @@ const ( compiledFolderName = "bin" ) -// newCompilingLifeCycle creates LifeCycle with compiled SDK environment. -func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, executableFileExtension string, compiledFileExtension string) *LifeCycle { +// newCompilingLifeCycle creates LifeCycle for compiled SDK environment. +func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, sourceFileExtension string, compiledFileExtension string) *LifeCycle { baseFileFolder := filepath.Join(workingDir, baseFileFolder, pipelineId.String()) srcFileFolder := filepath.Join(baseFileFolder, sourceFolderName) binFileFolder := filepath.Join(baseFileFolder, compiledFolderName) @@ -35,11 +35,11 @@ func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, executableFi folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ BaseFolder: baseFileFolder, - ExecutableFolder: srcFileFolder, + SourceFileFolder: srcFileFolder, CompiledFolder: binFileFolder, }, Extension: Extension{ - ExecutableExtension: executableFileExtension, + SourceFileExtension: sourceFileExtension, CompiledExtension: compiledFileExtension, }, pipelineId: pipelineId, diff --git a/playground/backend/internal/preparators/java_preparators_test.go b/playground/backend/internal/preparators/java_preparators_test.go index a34579798829..7bfeef92c671 100644 --- a/playground/backend/internal/preparators/java_preparators_test.go +++ b/playground/backend/internal/preparators/java_preparators_test.go @@ -36,7 +36,7 @@ func Test_replace(t *testing.T) { lc, _ := fs_tool.NewLifeCycle(pb.Sdk_SDK_JAVA, uuid.New(), filepath.Join(path, "temp")) _ = lc.CreateFolders() defer os.RemoveAll(filepath.Join(path, "temp")) - _, _ = lc.CreateExecutableFile(codeWithPublicClass) + _, _ = lc.CreateSourceCodeFile(codeWithPublicClass) type args struct { args []interface{} @@ -54,7 +54,7 @@ func Test_replace(t *testing.T) { }, { name: "original file exists", - args: args{[]interface{}{lc.GetAbsoluteExecutableFilePath(), classWithPublicModifierPattern, classWithoutPublicModifierPattern}}, + args: args{[]interface{}{lc.GetAbsoluteSourceFilePath(), classWithPublicModifierPattern, classWithoutPublicModifierPattern}}, wantCode: codeWithoutPublicClass, wantErr: false, }, diff --git a/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go b/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go index 63ae04ad61de..b13d7bff6c28 100644 --- a/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go +++ b/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go @@ -40,9 +40,9 @@ func Setup(sdk pb.Sdk, code string, pipelineId uuid.UUID, workingDir string) (*f } // create file with code - _, err = lc.CreateExecutableFile(code) + _, err = lc.CreateSourceCodeFile(code) if err != nil { - logger.Errorf("%s: RunCode(): CreateExecutableFile(): %s\n", pipelineId, err.Error()) + logger.Errorf("%s: RunCode(): CreateSourceCodeFile(): %s\n", pipelineId, err.Error()) return nil, err } return lc, nil From 6b7603d16f17eafc63e7e2f5ff59f9cdd342b48f Mon Sep 17 00:00:00 2001 From: daria-malkova Date: Mon, 22 Nov 2021 13:31:55 +0300 Subject: [PATCH 4/4] Reanme LF compile -> executables --- playground/backend/cmd/server/controller.go | 2 +- .../code_processing/code_processing_test.go | 2 +- .../internal/executors/executor_builder.go | 4 ++-- .../internal/executors/executor_test.go | 2 +- playground/backend/internal/fs_tool/fs.go | 22 +++++++++---------- .../backend/internal/fs_tool/fs_test.go | 22 +++++++++---------- playground/backend/internal/fs_tool/go_fs.go | 6 ++--- .../backend/internal/fs_tool/go_fs_test.go | 10 ++++----- .../backend/internal/fs_tool/java_fs_test.go | 18 +++++++-------- .../internal/fs_tool/lc_constructor.go | 10 ++++----- .../run_builder/run_builder_setuper.go | 6 ++--- .../run_builder/run_builder_setuper_test.go | 6 ++--- 12 files changed, 55 insertions(+), 55 deletions(-) diff --git a/playground/backend/cmd/server/controller.go b/playground/backend/cmd/server/controller.go index 5f63e6b56558..6c90474ee981 100644 --- a/playground/backend/cmd/server/controller.go +++ b/playground/backend/cmd/server/controller.go @@ -65,7 +65,7 @@ func (controller *playgroundController) RunCode(ctx context.Context, info *pb.Ru return nil, errors.InternalError("Run code", "Error during setup file system: "+err.Error()) } - compileBuilder, err := compile_builder.Setup(lc.GetAbsoluteSourceFilePath(), lc.GetAbsoluteExecutableFilesFolderPath(), info.Sdk, controller.env.BeamSdkEnvs.ExecutorConfig) + compileBuilder, err := compile_builder.Setup(lc.GetAbsoluteSourceFilePath(), lc.GetAbsoluteBaseFolderPath(), info.Sdk, controller.env.BeamSdkEnvs.ExecutorConfig) if err != nil { logger.Errorf("RunCode(): error during setup run builder: %s\n", err.Error()) return nil, errors.InvalidArgumentError("Run code", "Error during setup compile builder: "+err.Error()) diff --git a/playground/backend/internal/code_processing/code_processing_test.go b/playground/backend/internal/code_processing/code_processing_test.go index 5f0fb2ba8af2..9cb73c9d5404 100644 --- a/playground/backend/internal/code_processing/code_processing_test.go +++ b/playground/backend/internal/code_processing/code_processing_test.go @@ -223,7 +223,7 @@ func Test_Process(t *testing.T) { t.Run(tt.name, func(t *testing.T) { lc, _ := fs_tool.NewLifeCycle(pb.Sdk_SDK_JAVA, tt.args.pipelineId, os.Getenv("APP_WORK_DIR")) filePath := lc.GetAbsoluteSourceFilePath() - workingDir := lc.GetAbsoluteExecutableFilesFolderPath() + workingDir := lc.GetAbsoluteBaseFolderPath() exec := executors.NewExecutorBuilder(). WithValidator(). diff --git a/playground/backend/internal/executors/executor_builder.go b/playground/backend/internal/executors/executor_builder.go index 31f126cc567a..82a9f2e23410 100644 --- a/playground/backend/internal/executors/executor_builder.go +++ b/playground/backend/internal/executors/executor_builder.go @@ -120,8 +120,8 @@ func (b *RunBuilder) WithArgs(runArgs []string) *RunBuilder { return b } -//WithClassName adds file name to executor -func (b *RunBuilder) WithClassName(name string) *RunBuilder { +//WithExecutableName adds file name to executor +func (b *RunBuilder) WithExecutableName(name string) *RunBuilder { b.actions = append(b.actions, func(e *Executor) { e.runArgs.fileName = name }) diff --git a/playground/backend/internal/executors/executor_test.go b/playground/backend/internal/executors/executor_test.go index 977970bfc5b8..fbcfb525e6d5 100644 --- a/playground/backend/internal/executors/executor_test.go +++ b/playground/backend/internal/executors/executor_test.go @@ -51,7 +51,7 @@ func BaseExecutorBuilder(envs environment.BeamEnvs, workingDir string, filePath WithRunner(). WithCommand(envs.ExecutorConfig.RunCmd). WithArgs(envs.ExecutorConfig.RunArgs). - WithClassName("HelloWorld"). + WithExecutableName("HelloWorld"). WithWorkingDir(workingDir). WithValidator(). WithSdkValidators(validatorsFuncs). diff --git a/playground/backend/internal/fs_tool/fs.go b/playground/backend/internal/fs_tool/fs.go index 5e7ceaa22295..5d4721a67518 100644 --- a/playground/backend/internal/fs_tool/fs.go +++ b/playground/backend/internal/fs_tool/fs.go @@ -31,16 +31,16 @@ const ( // Folder contains names of folders with executable and compiled files. // For each SDK these values should be set depending on folders that need for the SDK. type Folder struct { - BaseFolder string - SourceFileFolder string - CompiledFolder string + BaseFolder string + SourceFileFolder string + ExecutableFileFolder string } // Extension contains executable and compiled files' extensions. // For each SDK these values should be set depending on SDK's extensions. type Extension struct { - SourceFileExtension string - CompiledExtension string + SourceFileExtension string + ExecutableFileExtension string } // LifeCycle is used for preparing folders and files to process code for one request. @@ -111,16 +111,16 @@ func (l *LifeCycle) GetAbsoluteSourceFilePath() string { return absoluteFilePath } -// GetAbsoluteBinaryFilePath returns absolute filepath to compiled file (/path/to/workingDir/executable_files/{pipelineId}/bin/{pipelineId}.{compiledExtension}). -func (l *LifeCycle) GetAbsoluteBinaryFilePath() string { - fileName := l.pipelineId.String() + l.Extension.CompiledExtension - filePath := filepath.Join(l.Folder.CompiledFolder, fileName) +// GetAbsoluteExecutableFilePath returns absolute filepath to compiled file (/path/to/workingDir/executable_files/{pipelineId}/bin/{pipelineId}.{executableExtension}). +func (l *LifeCycle) GetAbsoluteExecutableFilePath() string { + fileName := l.pipelineId.String() + l.Extension.ExecutableFileExtension + filePath := filepath.Join(l.Folder.ExecutableFileFolder, fileName) absoluteFilePath, _ := filepath.Abs(filePath) return absoluteFilePath } -// GetAbsoluteExecutableFilesFolderPath returns absolute path to executable folder (/path/to/workingDir/executable_files/{pipelineId}). -func (l *LifeCycle) GetAbsoluteExecutableFilesFolderPath() string { +// GetAbsoluteBaseFolderPath returns absolute path to executable folder (/path/to/workingDir/executable_files/{pipelineId}). +func (l *LifeCycle) GetAbsoluteBaseFolderPath() string { absoluteFilePath, _ := filepath.Abs(l.Folder.BaseFolder) return absoluteFilePath } diff --git a/playground/backend/internal/fs_tool/fs_test.go b/playground/backend/internal/fs_tool/fs_test.go index f87664fa9090..ef3b0a6f19c4 100644 --- a/playground/backend/internal/fs_tool/fs_test.go +++ b/playground/backend/internal/fs_tool/fs_test.go @@ -53,8 +53,8 @@ func TestLifeCycle_CreateExecutableFile(t *testing.T) { name: "executable folder doesn't exist", fields: fields{ folder: Folder{ - SourceFileFolder: srcFileFolder, - CompiledFolder: binFileFolder, + SourceFileFolder: srcFileFolder, + ExecutableFileFolder: binFileFolder, }, pipelineId: pipelineId, }, @@ -208,13 +208,13 @@ func TestNewLifeCycle(t *testing.T) { want: &LifeCycle{ folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ - BaseFolder: baseFileFolder, - SourceFileFolder: srcFileFolder, - CompiledFolder: binFileFolder, + BaseFolder: baseFileFolder, + SourceFileFolder: srcFileFolder, + ExecutableFileFolder: binFileFolder, }, Extension: Extension{ - SourceFileExtension: javaSourceFileExtension, - CompiledExtension: javaCompiledFileExtension, + SourceFileExtension: javaSourceFileExtension, + ExecutableFileExtension: javaCompiledFileExtension, }, ExecutableName: executableName, pipelineId: pipelineId, @@ -338,9 +338,9 @@ func TestLifeCycle_GetAbsoluteExecutableFilesFolderPath(t *testing.T) { Extension: tt.fields.Extension, pipelineId: tt.fields.pipelineId, } - got := l.GetAbsoluteExecutableFilesFolderPath() + got := l.GetAbsoluteBaseFolderPath() if got != tt.want { - t.Errorf("GetAbsoluteExecutableFilesFolderPath() got = %v, want %v", got, tt.want) + t.Errorf("GetAbsoluteBaseFolderPath() got = %v, want %v", got, tt.want) } }) } @@ -369,8 +369,8 @@ func TestLifeCycle_ExecutableName(t *testing.T) { name: "ExecutableName", fields: fields{ Folder: Folder{ - BaseFolder: baseFileFolder, - CompiledFolder: binFileFolder, + BaseFolder: baseFileFolder, + ExecutableFileFolder: binFileFolder, }, ExecutableName: func(u uuid.UUID, s string) (string, error) { return "MOCK_EXECUTABLE_NAME", nil diff --git a/playground/backend/internal/fs_tool/go_fs.go b/playground/backend/internal/fs_tool/go_fs.go index 597211f05023..1c642652fdea 100644 --- a/playground/backend/internal/fs_tool/go_fs.go +++ b/playground/backend/internal/fs_tool/go_fs.go @@ -20,11 +20,11 @@ import ( ) const ( - goExecutableFileExtension = ".go" - goCompiledFileExtension = "" + goSourceFileExtension = ".go" + goExecutableFileExtension = "" ) // newGoLifeCycle creates LifeCycle with go SDK environment. func newGoLifeCycle(pipelineId uuid.UUID, workingDir string) *LifeCycle { - return newCompilingLifeCycle(pipelineId, workingDir, goExecutableFileExtension, goCompiledFileExtension) + return newCompilingLifeCycle(pipelineId, workingDir, goSourceFileExtension, goExecutableFileExtension) } diff --git a/playground/backend/internal/fs_tool/go_fs_test.go b/playground/backend/internal/fs_tool/go_fs_test.go index 433d0b7aafbf..e026d1d9e199 100644 --- a/playground/backend/internal/fs_tool/go_fs_test.go +++ b/playground/backend/internal/fs_tool/go_fs_test.go @@ -49,13 +49,13 @@ func Test_newGoLifeCycle(t *testing.T) { want: &LifeCycle{ folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ - BaseFolder: baseFileFolder, - SourceFileFolder: srcFileFolder, - CompiledFolder: binFileFolder, + BaseFolder: baseFileFolder, + SourceFileFolder: srcFileFolder, + ExecutableFileFolder: binFileFolder, }, Extension: Extension{ - SourceFileExtension: goExecutableFileExtension, - CompiledExtension: goCompiledFileExtension, + SourceFileExtension: goSourceFileExtension, + ExecutableFileExtension: goExecutableFileExtension, }, pipelineId: pipelineId, }, diff --git a/playground/backend/internal/fs_tool/java_fs_test.go b/playground/backend/internal/fs_tool/java_fs_test.go index 175ea5fd19cf..a8baf8f9d5c0 100644 --- a/playground/backend/internal/fs_tool/java_fs_test.go +++ b/playground/backend/internal/fs_tool/java_fs_test.go @@ -51,13 +51,13 @@ func Test_newJavaLifeCycle(t *testing.T) { want: &LifeCycle{ folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ - BaseFolder: baseFileFolder, - SourceFileFolder: srcFileFolder, - CompiledFolder: binFileFolder, + BaseFolder: baseFileFolder, + SourceFileFolder: srcFileFolder, + ExecutableFileFolder: binFileFolder, }, Extension: Extension{ - SourceFileExtension: javaSourceFileExtension, - CompiledExtension: javaCompiledFileExtension, + SourceFileExtension: javaSourceFileExtension, + ExecutableFileExtension: javaCompiledFileExtension, }, ExecutableName: executableName, pipelineId: pipelineId, @@ -103,7 +103,7 @@ func Test_executableName(t *testing.T) { wantErr bool }{ { - // Test case with calling executableName method with correct pipelineId and workingDir. + // Test case with calling sourceFileName method with correct pipelineId and workingDir. // As a result, want to receive a name that should be executed name: "get executable name", prepare: func() { @@ -122,7 +122,7 @@ func Test_executableName(t *testing.T) { wantErr: false, }, { - // Test case with calling executableName method with correct pipelineId and workingDir. + // Test case with calling sourceFileName method with correct pipelineId and workingDir. // As a result, want to receive an error. name: "directory doesn't exist", prepare: func() {}, @@ -139,11 +139,11 @@ func Test_executableName(t *testing.T) { tt.prepare() got, err := executableName(tt.args.pipelineId, tt.args.workingDir) if (err != nil) != tt.wantErr { - t.Errorf("executableName() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("sourceFileName() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("executableName() got = %v, want %v", got, tt.want) + t.Errorf("sourceFileName() got = %v, want %v", got, tt.want) } }) } diff --git a/playground/backend/internal/fs_tool/lc_constructor.go b/playground/backend/internal/fs_tool/lc_constructor.go index dbadbca9beea..a979f825a68e 100644 --- a/playground/backend/internal/fs_tool/lc_constructor.go +++ b/playground/backend/internal/fs_tool/lc_constructor.go @@ -34,13 +34,13 @@ func newCompilingLifeCycle(pipelineId uuid.UUID, workingDir string, sourceFileEx return &LifeCycle{ folderGlobs: []string{baseFileFolder, srcFileFolder, binFileFolder}, Folder: Folder{ - BaseFolder: baseFileFolder, - SourceFileFolder: srcFileFolder, - CompiledFolder: binFileFolder, + BaseFolder: baseFileFolder, + SourceFileFolder: srcFileFolder, + ExecutableFileFolder: binFileFolder, }, Extension: Extension{ - SourceFileExtension: sourceFileExtension, - CompiledExtension: compiledFileExtension, + SourceFileExtension: sourceFileExtension, + ExecutableFileExtension: compiledFileExtension, }, pipelineId: pipelineId, } diff --git a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go index 09bea69437af..e3cc0e7c4903 100644 --- a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go +++ b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper.go @@ -31,7 +31,7 @@ func Setup(pipelineId uuid.UUID, lc *fs_tool.LifeCycle, workingDir string, sdkEn WithRunner(). WithCommand(sdkEnv.ExecutorConfig.RunCmd). WithArgs(sdkEnv.ExecutorConfig.RunArgs). - WithWorkingDir(lc.GetAbsoluteExecutableFilesFolderPath()) + WithWorkingDir(lc.GetAbsoluteBaseFolderPath()) switch sdkEnv.ApacheBeamSdk { case pb.Sdk_SDK_JAVA: @@ -42,9 +42,9 @@ func Setup(pipelineId uuid.UUID, lc *fs_tool.LifeCycle, workingDir string, sdkEn } runBuilder = runBuilder. - WithClassName(className) + WithExecutableName(className) case pb.Sdk_SDK_GO: - runBuilder = runBuilder.WithCommand(lc.GetAbsoluteBinaryFilePath()) + runBuilder = runBuilder.WithCommand(lc.GetAbsoluteExecutableFilePath()) default: return nil, fmt.Errorf("incorrect sdk: %s", sdkEnv.ApacheBeamSdk) } diff --git a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go index f1264847abe5..c07f7dc98dec 100644 --- a/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go +++ b/playground/backend/internal/setup_tools/run_builder/run_builder_setuper_test.go @@ -43,7 +43,7 @@ func TestSetup(t *testing.T) { if err != nil { panic(err) } - _, err = os.Create(successLc.Folder.CompiledFolder + "/temp.class") + _, err = os.Create(successLc.Folder.ExecutableFileFolder + "/temp.class") if err != nil { panic(err) } @@ -116,8 +116,8 @@ func TestSetup(t *testing.T) { WithRunner(). WithCommand(sdkEnv.ExecutorConfig.RunCmd). WithArgs(sdkEnv.ExecutorConfig.RunArgs). - WithWorkingDir(successLc.GetAbsoluteExecutableFilesFolderPath()). - WithClassName(className), + WithWorkingDir(successLc.GetAbsoluteBaseFolderPath()). + WithExecutableName(className), wantErr: false, }, }