From b73f5f70ac4184b56a0a03922731c5f2f69b9566 Mon Sep 17 00:00:00 2001 From: Sergey Kalinin <91209855+snkalinin@users.noreply.github.com> Date: Tue, 16 Nov 2021 14:59:11 +0300 Subject: [PATCH 1/5] [Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83) * Update workflows for playground * Attempt to fix tests * Remove continue on error to catch errors * Fix linter problem for backend dockerfile * Update folder to run backend go linter * Moved flutter test to execution via gradle tasks --- .../workflows/build_playground_backend.yml | 14 ++++++++----- .../workflows/build_playground_frontend.yml | 20 +++++++------------ playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index 96e8aae9b0d5..bd2868768246 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,10 +17,8 @@ name: Build And Deploy Playground Backend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -45,12 +43,14 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint - run: "golangci-lint run internal/..." +# run: "golangci-lint run internal/..." + run: "golangci-lint run cmd/server/..." working-directory: playground/backend/ - continue-on-error: true + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue + run: rm ~/.m2/settings.xml - name: Run Tests run: ./gradlew playground:backend:test - continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,12 +62,16 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index 9c2d56a52428..f93e330442b0 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,12 +17,10 @@ name: Build And Deploy Playground Frontend Application on: push: - branches: ['master', 'release-*'] tags: 'v*' pull_request: - branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/backend/**'] + paths: ['playground/frontend/**'] workflow_dispatch: jobs: @@ -43,18 +41,10 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' - - name: Prepare Flutter lint - run: "flutter pub add flutter_lints --dev" - working-directory: playground/frontend/ - continue-on-error: true - name: Run Lint - run: "flutter analyze" - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:analyze - name: Run Tests - run: flutter test - working-directory: playground/frontend/ - continue-on-error: true + run: ./gradlew playground:frontend:test - name: install npm uses: actions/setup-node@v2 with: @@ -66,12 +56,16 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json + if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' + if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend + if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index d7275020dcfa..8846368ff8b6 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE +ARG BASE_IMAGE=apache/beam_java8_sdk:latest FROM golang:1.17-buster AS build # Setup Go Environment From d449275b8ebd623760b34981f56c8f62b140ad9d Mon Sep 17 00:00:00 2001 From: Ilya Date: Tue, 16 Nov 2021 15:07:49 +0300 Subject: [PATCH 2/5] Revert "[Playground][BEAM-12941][Bugfix] Fix workflows for playground applications (#83)" (#88) This reverts commit b73f5f70ac4184b56a0a03922731c5f2f69b9566. --- .../workflows/build_playground_backend.yml | 14 +++++-------- .../workflows/build_playground_frontend.yml | 20 ++++++++++++------- playground/backend/containers/java/Dockerfile | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index bd2868768246..96e8aae9b0d5 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -17,8 +17,10 @@ name: Build And Deploy Playground Backend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -43,14 +45,12 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint -# run: "golangci-lint run internal/..." - run: "golangci-lint run cmd/server/..." + run: "golangci-lint run internal/..." working-directory: playground/backend/ - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: rm ~/.m2/settings.xml + continue-on-error: true - name: Run Tests run: ./gradlew playground:backend:test + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -62,16 +62,12 @@ jobs: working-directory: playground/backend/containers/java - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index f93e330442b0..9c2d56a52428 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -17,10 +17,12 @@ name: Build And Deploy Playground Frontend Application on: push: + branches: ['master', 'release-*'] tags: 'v*' pull_request: + branches: ['master', 'release-*'] tags: 'v*' - paths: ['playground/frontend/**'] + paths: ['playground/backend/**'] workflow_dispatch: jobs: @@ -41,10 +43,18 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' + - name: Prepare Flutter lint + run: "flutter pub add flutter_lints --dev" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Lint - run: ./gradlew playground:frontend:analyze + run: "flutter analyze" + working-directory: playground/frontend/ + continue-on-error: true - name: Run Tests - run: ./gradlew playground:frontend:test + run: flutter test + working-directory: playground/frontend/ + continue-on-error: true - name: install npm uses: actions/setup-node@v2 with: @@ -56,16 +66,12 @@ jobs: working-directory: playground/frontend - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - name: Login to Docker Registry run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - name: Preapre Build run: ./gradlew --debug playground:frontend:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' - if: startsWith(github.ref, 'ref/tags/') - name: Deploy Backend Application env: GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository" working-directory: playground/terraform/applications/backend - if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile index 8846368ff8b6..d7275020dcfa 100644 --- a/playground/backend/containers/java/Dockerfile +++ b/playground/backend/containers/java/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### -ARG BASE_IMAGE=apache/beam_java8_sdk:latest +ARG BASE_IMAGE FROM golang:1.17-buster AS build # Setup Go Environment From caed401d2b95e912728184de2cda6b916a1682af Mon Sep 17 00:00:00 2001 From: Sergey Kalinin Date: Thu, 18 Nov 2021 08:26:31 +0000 Subject: [PATCH 3/5] Create precommit tasks for backend and frontend, replaced separate tests in actions to running precommit --- .github/workflows/build_playground_backend.yml | 5 ++--- .github/workflows/build_playground_frontend.yml | 6 ++---- build.gradle.kts | 8 ++------ playground/backend/build.gradle.kts | 5 +++++ playground/frontend/build.gradle | 6 ++++++ 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index bd2868768246..0c4d95713ba0 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -43,14 +43,13 @@ jobs: - name: Prepare Go lint env run: "sudo ./playground/backend/env_setup.sh" - name: Run Lint -# run: "golangci-lint run internal/..." run: "golangci-lint run cmd/server/..." working-directory: playground/backend/ - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue run: rm ~/.m2/settings.xml - - name: Run Tests - run: ./gradlew playground:backend:test + - name: Run PreCommit + run: ./gradlew playground:backend:precommit - name: install npm uses: actions/setup-node@v2 with: diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index f93e330442b0..44148264165d 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -41,10 +41,8 @@ jobs: uses: subosito/flutter-action@v1 with: channel: 'stable' - - name: Run Lint - run: ./gradlew playground:frontend:analyze - - name: Run Tests - run: ./gradlew playground:frontend:test + - name: Run PreCommit + run: ./gradlew playground:frontend:precommit - name: install npm uses: actions/setup-node@v2 with: diff --git a/build.gradle.kts b/build.gradle.kts index 25029151eaa2..37e0164a6749 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -248,12 +248,8 @@ task("goIntegrationTests") { } task("playgroundPreCommit") { - dependsOn(":playground:backend:tidy") - dependsOn(":playground:backend:test") - - dependsOn(":playground:frontend:pubGet") - dependsOn(":playground:frontend:analyze") - dependsOn(":playground:frontend:test") + dependsOn(":playground:backend:precommit") + dependsOn(":playground:frontend:precommit") } task("pythonPreCommit") { diff --git a/playground/backend/build.gradle.kts b/playground/backend/build.gradle.kts index 0547b7cb1c77..35ba5bc8edeb 100644 --- a/playground/backend/build.gradle.kts +++ b/playground/backend/build.gradle.kts @@ -50,3 +50,8 @@ task("test") { } } } + +task("precommit") { + dependsOn(":playground:backend:tidy") + dependsOn(":playground:backend:test") +} diff --git a/playground/frontend/build.gradle b/playground/frontend/build.gradle index 98e8b929e016..c46f60511635 100644 --- a/playground/frontend/build.gradle +++ b/playground/frontend/build.gradle @@ -92,6 +92,12 @@ task test { } } +task precommit { + dependsOn(":playground:frontend:pubGet") + dependsOn(":playground:frontend:analyze") + dependsOn(":playground:frontend:test") +} + task copyDockerfileDependencies(type: Copy) { group = "build" description = "Copy files that required to build docker container" From 4c042690b5e0e8c9b44d86c8486916c0c222e59e Mon Sep 17 00:00:00 2001 From: Sergey Kalinin Date: Thu, 18 Nov 2021 18:24:37 +0000 Subject: [PATCH 4/5] Added linter to preCommit, updated install terraform via github actions --- .../workflows/build_playground_backend.yml | 10 ++++----- playground/backend/build.gradle.kts | 20 +++++++++++++++++ playground/backend/env_setup.sh | 22 ++++++++++++------- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index 0c4d95713ba0..4b6ce042816c 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -30,6 +30,7 @@ jobs: env: GO_VERSION: 1.17.0 BEAM_VERSION: 2.33.0 + TERRAFOM_VERSION: 1.0.9 steps: - name: Check out the repo uses: actions/checkout@v2 @@ -40,11 +41,6 @@ jobs: - uses: actions/setup-go@v2 with: go-version: '${{ env.GO_VERSION }}' - - name: Prepare Go lint env - run: "sudo ./playground/backend/env_setup.sh" - - name: Run Lint - run: "golangci-lint run cmd/server/..." - working-directory: playground/backend/ - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue run: rm ~/.m2/settings.xml @@ -59,6 +55,10 @@ jobs: - name: lint dockerfile run: dockerlint Dockerfile working-directory: playground/backend/containers/java + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} + if: startsWith(github.ref, 'ref/tags/') - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json if: startsWith(github.ref, 'ref/tags/') diff --git a/playground/backend/build.gradle.kts b/playground/backend/build.gradle.kts index 35ba5bc8edeb..e73b00e9b532 100644 --- a/playground/backend/build.gradle.kts +++ b/playground/backend/build.gradle.kts @@ -51,7 +51,27 @@ task("test") { } } +task("installLinter") { + doLast { + exec { + executable("sh") + args("env_setup.sh") + } + } +} + +task("runLint") { + dependsOn(":playground:backend:installLinter") + doLast { + exec { + executable("golangci-lint") + args("run", "cmd/server/...") + } + } +} task("precommit") { + dependsOn(":playground:backend:runLint") dependsOn(":playground:backend:tidy") dependsOn(":playground:backend:test") } + diff --git a/playground/backend/env_setup.sh b/playground/backend/env_setup.sh index f12af691f739..1b87b319bcf9 100755 --- a/playground/backend/env_setup.sh +++ b/playground/backend/env_setup.sh @@ -17,12 +17,18 @@ GO_LINTER_VERSION=1.42.1 # Install GO Linter -wget https://github.com/golangci/golangci-lint/releases/download/v1.42.1/golangci-lint-$GO_LINTER_VERSION-linux-amd64.deb -dpkg -i golangci-lint-$GO_LINTER_VERSION-linux-amd64.deb +#wget https://github.com/golangci/golangci-lint/releases/download/v1.42.1/golangci-lint-$GO_LINTER_VERSION-linux-amd64.deb +#dpkg -i golangci-lint-$GO_LINTER_VERSION-linux-amd64.deb -# Install Terraform -apt-get install -y gnupg software-properties-common curl -curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - -apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" -apt-get update -apt-get install -y terraform +kernelname=$(uname -s) + +# Running on Linux +if [ "$kernelname" = "Linux" ]; then + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v$GO_LINTER_VERSION + +# Running on Mac +elif [ "$kernelname" = "Darwin" ]; then + brew tap golangci/tap + brew install golangci/tap/golangci-lint +else echo "Unrecognized Kernel Name: $kernelname" +fi From 50bf13acc1919f54c1050fde3876c00c5078689c Mon Sep 17 00:00:00 2001 From: Sergey Kalinin Date: Mon, 22 Nov 2021 17:54:16 +0000 Subject: [PATCH 5/5] Fix typos and install of terraform --- .github/workflows/build_playground_backend.yml | 3 +-- .github/workflows/build_playground_frontend.yml | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml index 4b6ce042816c..48eab26b2a9e 100644 --- a/.github/workflows/build_playground_backend.yml +++ b/.github/workflows/build_playground_backend.yml @@ -19,7 +19,6 @@ on: push: tags: 'v*' pull_request: - tags: 'v*' paths: ['playground/backend/**'] workflow_dispatch: @@ -30,7 +29,7 @@ jobs: env: GO_VERSION: 1.17.0 BEAM_VERSION: 2.33.0 - TERRAFOM_VERSION: 1.0.9 + TERRAFORM_VERSION: 1.0.9 steps: - name: Check out the repo uses: actions/checkout@v2 diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml index 44148264165d..ee1bed8f2b5a 100644 --- a/.github/workflows/build_playground_frontend.yml +++ b/.github/workflows/build_playground_frontend.yml @@ -19,7 +19,6 @@ on: push: tags: 'v*' pull_request: - tags: 'v*' paths: ['playground/frontend/**'] workflow_dispatch: @@ -30,6 +29,7 @@ jobs: env: GO_VERSION: 1.17.0 BEAM_VERSION: 2.33.0 + TERRAFORM_VERSION: 1.0.9 steps: - name: Check out the repo uses: actions/checkout@v2 @@ -52,6 +52,10 @@ jobs: - name: lint dockerfile run: dockerlint Dockerfile working-directory: playground/frontend + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} + if: startsWith(github.ref, 'ref/tags/') - name: Setup GCP account run: echo ${{ secrets.GCP_ACCESS_KEY }} | base64 -d > /tmp/gcp_access.json if: startsWith(github.ref, 'ref/tags/')