From 183c513507e2d34f929e56dc2d73b1b4075a5487 Mon Sep 17 00:00:00 2001 From: Sergey Kalinin Date: Mon, 17 Jan 2022 20:38:45 +0000 Subject: [PATCH 1/2] Fix deploy of examples --- .../workflows/playground_deploy_examples.yml | 27 +++++++++++++------ playground/infrastructure/helm/Chart.yaml | 2 +- .../helm/templates/service-python.yaml | 2 +- .../helm/templates/serviceaccount.yaml | 10 +++++++ 4 files changed, 31 insertions(+), 10 deletions(-) create mode 100644 playground/infrastructure/helm/templates/serviceaccount.yaml diff --git a/.github/workflows/playground_deploy_examples.yml b/.github/workflows/playground_deploy_examples.yml index 641e87385ee6..d27065195fa0 100644 --- a/.github/workflows/playground_deploy_examples.yml +++ b/.github/workflows/playground_deploy_examples.yml @@ -18,6 +18,7 @@ name: Collect And Deploy Playground Examples on: push: tags: 'v*' + branch: 'beta-version' workflow_dispatch: jobs: @@ -28,7 +29,6 @@ jobs: BEAM_ROOT_DIR: ../../ BEAM_EXAMPLE_CATEGORIES: ../categories.yaml BEAM_VERSION: 2.33.0 - SERVER_ADDRESS: backend-python-dot-datatokenization.uc.r.appspot.com K8S_NAMESPACE: playground-backend HELM_APP_NAME: playground-backend steps: @@ -56,24 +56,34 @@ jobs: - name: install deps run: pip install -r requirements.txt working-directory: playground/infrastructure + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue + run: rm ~/.m2/settings.xml - name: Setup GCP account run: | - echo "${{ secrets.GCP_ACCESS_KEY }}" | base64 -d > /tmp/gcp_access.json - gcloud auth activate-service-account --project=datatokenization --key-file=/tmp/gcp_access.json + echo "${{ secrets.PLAYGROUND_GCP_ACCESS_KEY }}" | base64 -d > /tmp/gcp_access.json + which gcloud + gcloud auth activate-service-account --project=apache-beam-testing --key-file=/tmp/gcp_access.json + - name: Set Docker Tag + run: echo "DOCKERTAG=${GITHUB_SHA}" >> $GITHUB_ENV + - name: Set Docker Tag If Github Tag was trigger + run: echo "DOCKERTAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + if: startsWith(github.ref, 'refs/tags/') - name: Get K8s Config run: gcloud container clusters get-credentials --region us-central1-a playground-examples - name: Login to Docker Registry - run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }} + run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.PLAYGROUND_REGISTRY_NAME }} - name: Build And Push Java Backend - run: ./gradlew playground:backend:containers:java:dockerTagPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' -Pdocker-tag="$GITHUB_SHA" + run: | + ./gradlew playground:backend:containers:java:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' -Pdocker-tag=${{ env.DOCKERTAG }} - name: Build And Push Go Backend - run: ./gradlew playground:backend:containers:go:dockerTagPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pdocker-tag="$GITHUB_SHA" + run: ./gradlew playground:backend:containers:go:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }} - name: Build And Push Python Backend - run: ./gradlew playground:backend:containers:python:dockerTagPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pdocker-tag="$GITHUB_SHA" + run: ./gradlew playground:backend:containers:python:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }} - name: Install helm chart run: | kubectl create namespace $K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - &&\ - helm install --namespace $K8S_NAMESPACE $HELM_APP_NAME . --set global.registry="${{ secrets.REGISTRY_NAME }}/${{ secrets.PROJECT_ID }}/playground-repository" --set global.tag="$GITHUB_SHA" &&\ + helm install --namespace $K8S_NAMESPACE $HELM_APP_NAME . --set global.registry="${{ secrets.PLAYGROUND_REGISTRY_NAME }}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository" --set global.tag=${{ env.DOCKERTAG }} &&\ sleep 120 working-directory: playground/infrastructure/helm - name: Run Python Examples CI @@ -81,6 +91,7 @@ jobs: K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') BEAM_ROOT_DIR="../../" BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml" + echo $K8S_SERVER_ADDRESS export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" && python3 ci_cd.py --step CI --sdk SDK_PYTHON working-directory: playground/infrastructure - name: Run Python Examples CD diff --git a/playground/infrastructure/helm/Chart.yaml b/playground/infrastructure/helm/Chart.yaml index 55b0ca562486..ac96c361411e 100644 --- a/playground/infrastructure/helm/Chart.yaml +++ b/playground/infrastructure/helm/Chart.yaml @@ -16,5 +16,5 @@ apiVersion: v2 name: playground description: A Helm chart for Apache Beam Application Deployment type: application -version: beta +version: 0.0.1 appVersion: "1.16.0" diff --git a/playground/infrastructure/helm/templates/service-python.yaml b/playground/infrastructure/helm/templates/service-python.yaml index 5dbb47dec7c4..b220dcd11cf9 100644 --- a/playground/infrastructure/helm/templates/service-python.yaml +++ b/playground/infrastructure/helm/templates/service-python.yaml @@ -19,7 +19,7 @@ metadata: labels: app: service-python annotations: - cloud.google.com/neg: '{"exposed_ports": {"8081":{}}}' + cloud.google.com/neg: '{"exposed_ports": {"8080":{}}}' spec: ports: - name: 8080-8081 diff --git a/playground/infrastructure/helm/templates/serviceaccount.yaml b/playground/infrastructure/helm/templates/serviceaccount.yaml new file mode 100644 index 000000000000..2a3b07070272 --- /dev/null +++ b/playground/infrastructure/helm/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "playground.serviceAccountName" . }} + labels: + {{- include "playground.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} From 7d5c3a0be763577dba8826c0c20fee08cc1fbd83 Mon Sep 17 00:00:00 2001 From: Sergey Kalinin Date: Mon, 17 Jan 2022 20:51:38 +0000 Subject: [PATCH 2/2] Update license --- .../helm/templates/serviceaccount.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/playground/infrastructure/helm/templates/serviceaccount.yaml b/playground/infrastructure/helm/templates/serviceaccount.yaml index 2a3b07070272..fb4c05e74ee2 100644 --- a/playground/infrastructure/helm/templates/serviceaccount.yaml +++ b/playground/infrastructure/helm/templates/serviceaccount.yaml @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. apiVersion: v1 kind: ServiceAccount metadata: