From 59ddccfe9869d3b84f8f20908fa22de4e05250cf Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 10 Mar 2023 17:40:10 +0500 Subject: [PATCH 01/35] Added Terraform scripts for TOB infra --- learning/tour-of-beam/terraform/README.md | 121 ++++++ .../tour-of-beam/terraform/api_enable/main.tf | 29 ++ .../terraform/api_enable/variables.tf | 20 + .../tour-of-beam/terraform/build.gradle.kts | 388 ++++++++++++++++++ .../terraform/cloud_functions/main.tf | 59 +++ .../terraform/cloud_functions/output.tf | 20 + .../terraform/cloud_functions/variables.tf | 46 +++ .../environment/test/state.tfbackend | 20 + .../terraform/functions_buckets/main.tf | 32 ++ .../terraform/functions_buckets/output.tf | 28 ++ .../terraform/functions_buckets/variables.tf | 31 ++ learning/tour-of-beam/terraform/main.tf | 48 +++ learning/tour-of-beam/terraform/output.tf | 32 ++ learning/tour-of-beam/terraform/provider.tf | 35 ++ learning/tour-of-beam/terraform/setup/data.tf | 3 + learning/tour-of-beam/terraform/setup/iam.tf | 41 ++ .../tour-of-beam/terraform/setup/output.tf | 20 + .../tour-of-beam/terraform/setup/variables.tf | 28 ++ learning/tour-of-beam/terraform/variables.tf | 41 ++ 19 files changed, 1042 insertions(+) create mode 100644 learning/tour-of-beam/terraform/README.md create mode 100644 learning/tour-of-beam/terraform/api_enable/main.tf create mode 100644 learning/tour-of-beam/terraform/api_enable/variables.tf create mode 100644 learning/tour-of-beam/terraform/build.gradle.kts create mode 100644 learning/tour-of-beam/terraform/cloud_functions/main.tf create mode 100644 learning/tour-of-beam/terraform/cloud_functions/output.tf create mode 100644 learning/tour-of-beam/terraform/cloud_functions/variables.tf create mode 100644 learning/tour-of-beam/terraform/environment/test/state.tfbackend create mode 100644 learning/tour-of-beam/terraform/functions_buckets/main.tf create mode 100644 learning/tour-of-beam/terraform/functions_buckets/output.tf create mode 100644 learning/tour-of-beam/terraform/functions_buckets/variables.tf create mode 100644 learning/tour-of-beam/terraform/main.tf create mode 100644 learning/tour-of-beam/terraform/output.tf create mode 100644 learning/tour-of-beam/terraform/provider.tf create mode 100644 learning/tour-of-beam/terraform/setup/data.tf create mode 100644 learning/tour-of-beam/terraform/setup/iam.tf create mode 100644 learning/tour-of-beam/terraform/setup/output.tf create mode 100644 learning/tour-of-beam/terraform/setup/variables.tf create mode 100644 learning/tour-of-beam/terraform/variables.tf diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md new file mode 100644 index 000000000000..3b29bf072fe4 --- /dev/null +++ b/learning/tour-of-beam/terraform/README.md @@ -0,0 +1,121 @@ + +# The Tour of Beam deployment on GCP +This guide provides instructions on how to deploy the Tour of Beam environment on Google Cloud Platform (GCP) and Firebase environment. Before starting the deployment, ensure that you have the following prerequisites in place: + +## Prerequisites: + +1. [GCP project](https://cloud.google.com/resource-manager/docs/creating-managing-projects) +2. [GCP User account](https://cloud.google.com/appengine/docs/standard/access-control?tab=python) _(Note: You will find the instruction "How to create User account" for your new project)_
+ Ensure that the account has at least following privileges: + - Cloud Datastore Owner + - Create Service Accounts + - Security Admin + - Service Account User + - Service Usage Admin + - Storage Admin + - Kubernetes Engine Cluster Viewer + +3. [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for saving deployment state + +4. An OS with the following software installed: + +* [Java](https://adoptopenjdk.net/) +* [NodeJS & npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm/) +* [Flutter (3.7.3 >)](https://docs.flutter.dev/get-started/install) +* [Dart SDK (2.19.2)](https://dart.dev/get-dart) +* [Firebase-tools CLI](https://www.npmjs.com/package/firebase-tools) +* [Terraform](https://www.terraform.io/downloads) +* [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) +* [Kubectl authentication plugin](https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke) +* [Go](https://go.dev/doc/install) + +5. Apache Beam Git repository cloned locally + +# Prepare deployment configuration: +Tour of Beam backend uses `terraform.tfvars` located in `learning/tour-of-beam/terraform/environment/environment_name/` to define variables specific to an environment (e.g., prod, test, staging). Follow the steps below to prepare the deployment configuration:
+1. Create a folder (referred to as `environment_name`) to define a new environment and place configuration files into it: + +* `terraform.tfvars` environment variables: +``` +project_id = "gcp_project_id" # Your GCP Project ID +cloudfunctions_bucket = "gcs_bucket_name" # Globally unique name of the bucket to store cloud functions' source code +region = "gcp_region" # Your GCP resources region +service_account_id = "service_account_name" # Name of GCP service account to run Tour of Beam cloud functions + +``` +* `state.tfbackend` environment variables: +``` +bucket = "bucket_name" # Your created bucket name for terraform tfstate file +``` +2. Configure authentication for the Google Cloud Platform (GCP) +``` +gcloud init +``` +``` +gcloud auth application-default login +``` + +3. Configure authentication in the GCP Docker registry: +``` + gcloud auth configure-docker `chosen_region`-docker.pkg.dev +``` +4. And the authentication in GCP Google Kubernetes Engine: +``` +gcloud container clusters get-credentials --region `chosen_gke_zone` `gke_name` --project `project_id` +``` + +# Deploy the Tour of Beam Backend: + +5. Run the following command from the top-level repository folder ("beam") to deploy the Tour of Beam Backend infrastructure: +``` +./gradlew learning:tour-of-beam:terraform:InitBackend -Pproject_environment="environment_name" -Pproject_id="gcp-project-id" -Pgcloud_account=`gcloud config get-value core/account` +``` +Where: +- **project_environment** - environment name +- **project_id** - name of your GCP Project ID + +# Deploy the Tour of Beam Frontend: + +6. Run the following command and follow the instructions to configure authentication for Firebase: +``` +firebase login --no-localhost +``` + +7. Run the following command from the top-level repository folder ("beam") to deploy the Tour of Beam Frontend infrastructure: +``` +./gradlew learning:tour-of-beam:terraform:InitFrontend -Pproject_environment="environment_name" -Pproject_id="gcp-project-id" -Pdns-name="playground-dns-name" -Pregion="gcp-region" -Pwebapp_id="firebase_webapp_name" +``` +Where: +- **project_environment** - environment name +- **project_id** - name of your GCP Project ID +- **dns-name** - DNS name reserved for Beam Playground +- **region** - name of your GCP Resources region +- **webapp_id** - name of your Firebase Web Application that will be created (example: Tour-of-Beam-Web-App) + +# Validate the deployment of the Tour of Beam: +8. Open the Tour of Beam webpage in a web browser (Hosting URL will be provided in terminal output) to ensure that deployment has been successfully completed. + +Example: +``` +✔ Deploy complete! + +Project Console: https://console.firebase.google.com/project/some-gcp-project-id/overview +Hosting URL: https://some-gcp-project-id.web.app +``` \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/api_enable/main.tf b/learning/tour-of-beam/terraform/api_enable/main.tf new file mode 100644 index 000000000000..3a61a090fe9e --- /dev/null +++ b/learning/tour-of-beam/terraform/api_enable/main.tf @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +resource "google_project_service" "required_services" { + project = var.project_id + for_each = toset([ + "cloudresourcemanager", + "iam", + "cloudbuild", + "cloudfunctions", + "firebase" + ]) + service = "${each.key}.googleapis.com" + disable_on_destroy = false +} diff --git a/learning/tour-of-beam/terraform/api_enable/variables.tf b/learning/tour-of-beam/terraform/api_enable/variables.tf new file mode 100644 index 000000000000..f4a375110500 --- /dev/null +++ b/learning/tour-of-beam/terraform/api_enable/variables.tf @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} diff --git a/learning/tour-of-beam/terraform/build.gradle.kts b/learning/tour-of-beam/terraform/build.gradle.kts new file mode 100644 index 000000000000..4406861d4ed4 --- /dev/null +++ b/learning/tour-of-beam/terraform/build.gradle.kts @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.pswidersk.gradle.terraform.TerraformTask +import java.io.ByteArrayOutputStream +import java.util.regex.Pattern + +plugins { + id("com.pswidersk.terraform-plugin") version "1.0.0" +} + +terraformPlugin { + terraformVersion.set("1.0.9") +} + +/* init Infrastructure for migrate */ +tasks.register("terraformInit") { + // exec args can be passed by commandline, for example + var environment = project.property("project_environment") as String + args( + "init", "-migrate-state", + "-backend-config=./environment/$environment/state.tfbackend", + "-var=environment=$environment", + if (file("./environment/$environment/terraform.tfvars").exists()) { + "-var-file=./environment/$environment/terraform.tfvars" + } else { + "-no-color" + } + ) + } + + /* refresh Infrastucture for remote state */ +tasks.register("terraformRef") { + var environment = project.property("project_environment") as String + args( + "refresh", + "-lock=false", + "-var=environment=$environment", + if (file("./environment/$environment/terraform.tfvars").exists()) { + "-var-file=./environment/$environment/terraform.tfvars" + } else { + "-no-color" + } + ) + } + +tasks.register("terraformApplyBackend") { + group = "backend-deploy" + var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String + var environment = project.property("project_environment") as String + var gcloud_account = project.property("gcloud_account") as String + args( + "apply", + "-auto-approve", + "-lock=false", + "-parallelism=3", + "-var=pg_router_host=$pg_router_host", + "-var=gcloud_init_account=$gcloud_account", + "-var=environment=$environment", + if (file("./environment/$environment/terraform.tfvars").exists()) { + "-var-file=./environment/$environment/terraform.tfvars" + } else { + "-no-color" + } + ) + tasks.getByName("uploadLearningMaterials").mustRunAfter(this) + + } + +tasks.register("terraformDestroy") { + var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String + var environment = project.property("project_environment") as String + var gcloud_account = project.property("gcloud_account") as String + args( + "destroy", + "-auto-approve", + "-lock=false", + "-var=pg_router_host=$pg_router_host", + "-var=environment=$environment", + "-var=gcloud_init_account=$gcloud_account", + if (file("./environment/$environment/terraform.tfvars").exists()) { + "-var-file=./environment/$environment/terraform.tfvars" + } else { + "-no-color" + } + ) +} + +tasks.register("getRouterHost") { + group = "backend-deploy" + val result = ByteArrayOutputStream() + exec { + commandLine("kubectl", "get", "svc", "-l", "app=backend-router-grpc", "-o", "jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}'") + standardOutput = result + } + val pg_router_host = result.toString().trim().replace("'", "") + project.extensions.extraProperties["pg_router_host"] = pg_router_host +} + +tasks.register("indexcreate") { + group = "backend-deploy" + val indexpath = "../backend/internal/storage/index.yaml" + exec { + executable("gcloud") + args("datastore", "indexes", "create", indexpath) + } +} + +tasks.register("firebaseProjectCreate") { + group = "frontend-deploy" + val result = ByteArrayOutputStream() + var project_id = project.property("project_id") as String + exec { + executable("firebase") + args("projects:list") + standardOutput = result + } + val output = result.toString().trim() + if (output.contains(project_id)) { + println("Firebase is already added to project $project_id.") + } else { + exec { + executable("firebase") + args("projects:addfirebase", project_id) + }.assertNormalExitValue() + println("Firebase has been added to project $project_id.") + } +} + +tasks.register("firebaseWebAppCreate") { + group = "frontend-deploy" + val result = ByteArrayOutputStream() + var project_id = project.property("project_id") as String + var webapp_id = project.property("webapp_id") as String + exec { + executable("firebase") + args("apps:list", "--project", project_id) + standardOutput = result + } + println(result) + val output = result.toString() + if (output.contains(webapp_id)) { + println("Webapp id $webapp_id is already created on the project: $project_id.") + val regex = Regex("$webapp_id[│ ]+([\\w:]+)[│ ]+WEB[│ ]+") + val firebaseAppId = regex.find(output)?.groupValues?.get(1)?.trim() + project.extensions.extraProperties["firebaseAppId"] = firebaseAppId + } else { + val result2 = ByteArrayOutputStream() + exec { + executable("firebase") + args("apps:create", "WEB", webapp_id, "--project", project_id) + standardOutput = result2 + }.assertNormalExitValue() + val firebaseAppId = result2.toString().lines().find { it.startsWith(" - App ID:") }?.substringAfter(":")?.trim() + project.extensions.extraProperties["firebaseAppId"] = firebaseAppId + println("Firebase app ID for newly created Firebase Web App: $firebaseAppId") + } +} + +// firebase apps:sdkconfig WEB AppId +tasks.register("getSdkConfigWebApp") { + group = "frontend-deploy" + val firebaseAppId = project.extensions.extraProperties["firebaseAppId"] as String + val result = ByteArrayOutputStream() + exec { + executable("firebase") + args("apps:sdkconfig", "WEB", firebaseAppId) + standardOutput = result + } + val output = result.toString().trim() + val pattern = Pattern.compile("\\{[^{]*\"locationId\":\\s*\".*?\"[^}]*\\}", Pattern.DOTALL) + val matcher = pattern.matcher(output) + if (matcher.find()) { + val firebaseConfigData = matcher.group().replace("{", "") + .replace("}", "") + .replace("\"locationId\":\\s*\".*?\",?".toRegex(), "") + .replace("\"(\\w+)\":".toRegex(), "$1:") + .replace(":\\s*\"(.*?)\"".toRegex(), ":\"$1\"") + project.extensions.extraProperties["firebaseConfigData"] = firebaseConfigData.trim() + println("Firebase config data: $firebaseConfigData") + } +} + +tasks.register("prepareFirebaseOptionsDart") { + group = "frontend-deploy" + val firebaseConfigData = project.extensions.extraProperties["firebaseConfigData"] as String + val file = project.file("../frontend/lib/firebase_options.dart") + val content = file.readText() + val updatedContent = content.replace(Regex("""static const FirebaseOptions web = FirebaseOptions\(([^)]+)\);"""), "static const FirebaseOptions web = FirebaseOptions(${firebaseConfigData});") + file.writeText(updatedContent) +} + +tasks.register("flutterPubGetPG") { + exec { + executable("flutter") + args("pub", "get") + workingDir("../../../playground/frontend/playground_components") + } +} + +tasks.register("flutterPubRunPG") { + exec { + executable("flutter") + args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") + workingDir("../../../playground/frontend/playground_components") + } +} + +tasks.register("flutterPubGetTob") { + exec { + executable("flutter") + args("pub", "get") + workingDir("../frontend") + } +} + +tasks.register("flutterPubRunTob") { + exec { + executable("flutter") + args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") + workingDir("../frontend") + } +} + +tasks.register("flutterBuildWeb") { + exec { + executable("flutter") + args("build", "web", "--profile", "--dart-define=Dart2jsOptimization=O0") + workingDir("../frontend") + } +} + +tasks.register("firebaseDeploy") { + var project_id = project.property("project_id") as String + exec { + commandLine("firebase", "deploy", "--project", project_id) + workingDir("../frontend") + } +} + +tasks.register("prepareConfig") { + group = "frontend-deploy" + var region = project.property("region") as String + var project_id = project.property("project_id") as String + var environment = project.property("project_environment") as String + var dns_name = project.property("dns-name") as String + val configFileName = "config.dart" + val modulePath = project(":learning:tour-of-beam:frontend").projectDir.absolutePath + val file = File("$modulePath/lib/$configFileName") + + file.writeText( + """ +const _cloudFunctionsProjectRegion = '$region'; +const _cloudFunctionsProjectId = '$project_id'; +const cloudFunctionsBaseUrl = 'https://' + '$region-$project_id' + '.cloudfunctions.net/${environment}_'; + + +const String kAnalyticsUA = 'UA-73650088-2'; +const String kApiClientURL = +'https://router.${dns_name}'; +const String kApiJavaClientURL = +'https://java.${dns_name}'; +const String kApiGoClientURL = +'https://go.${dns_name}'; +const String kApiPythonClientURL = +'https://python.${dns_name}'; +const String kApiScioClientURL = +'https://scio.${dns_name}'; +""" + ) +} + +tasks.register("prepareFirebasercConfig") { + group = "frontend-deploy" + var project_id = project.property("project_id") as String + val configFileName = ".firebaserc" + val modulePath = project(":learning:tour-of-beam:frontend").projectDir.absolutePath + val file = File("$modulePath/$configFileName") + + file.writeText( + """ +{ + "projects": { + "default": "$project_id" + } +} +""" + ) +} + +tasks.register("uploadLearningMaterials") { + var project_id = project.property("project_id") as String + group = "backend-deploy" + exec { + commandLine("go", "run", "cmd/ci_cd/ci_cd.go") + environment("DATASTORE_PROJECT_ID", project_id) + environment("GOOGLE_PROJECT_ID", project_id) + environment("TOB_LEARNING_ROOT", "../learning-content/") + workingDir("../backend") + } + dependsOn("terraformApplyBackend") + mustRunAfter("terraformApplyBackend") +} + +/* Tour of Beam backend init */ +tasks.register("InitBackend") { + group = "backend-deploy" + val getRouterHost = tasks.getByName("getRouterHost") + val indexCreate = tasks.getByName("indexcreate") + val tfInit = tasks.getByName("terraformInit") + val tfApplyBackend = tasks.getByName("terraformApplyBackend") + val uploadLearningMaterials = tasks.getByName("uploadLearningMaterials") + dependsOn(getRouterHost) + dependsOn(indexCreate) + dependsOn(tfInit) + dependsOn(tfApplyBackend) + dependsOn(uploadLearningMaterials) + indexCreate.mustRunAfter(getRouterHost) + tfInit.mustRunAfter(indexCreate) + tfApplyBackend.mustRunAfter(tfInit) + uploadLearningMaterials.mustRunAfter(tfApplyBackend) + +} + +tasks.register("DestroyBackend") { + group = "backend-destroy" + val getRouterHost = tasks.getByName("getRouterHost") + val terraformDestroy = tasks.getByName("terraformDestroy") + dependsOn(getRouterHost) + dependsOn(terraformDestroy) + terraformDestroy.mustRunAfter(getRouterHost) +} + +tasks.register("InitFrontend") { + group = "frontend-deploy" + val prepareConfig = tasks.getByName("prepareConfig") + val prepareFirebasercConfig = tasks.getByName("prepareFirebasercConfig") + val firebaseProjectCreate = tasks.getByName("firebaseProjectCreate") + val firebaseWebAppCreate = tasks.getByName("firebaseWebAppCreate") + val getSdkConfigWebApp = tasks.getByName("getSdkConfigWebApp") + val prepareFirebaseOptionsDart = tasks.getByName("prepareFirebaseOptionsDart") + val flutterPubGetPG = tasks.getByName("flutterPubGetPG") + val flutterPubRunPG = tasks.getByName("flutterPubRunPG") + val flutterPubGetTob = tasks.getByName("flutterPubGetTob") + val flutterPubRunTob = tasks.getByName("flutterPubRunTob") + val flutterBuildWeb = tasks.getByName("flutterBuildWeb") + val firebaseDeploy = tasks.getByName("firebaseDeploy") + dependsOn(prepareConfig) + dependsOn(prepareFirebasercConfig) + dependsOn(firebaseProjectCreate) + dependsOn(firebaseWebAppCreate) + dependsOn(getSdkConfigWebApp) + dependsOn(prepareFirebaseOptionsDart) + dependsOn(flutterPubGetPG) + dependsOn(flutterPubRunPG) + dependsOn(flutterPubGetTob) + dependsOn(flutterPubRunTob) + dependsOn(flutterBuildWeb) + dependsOn(firebaseDeploy) + prepareFirebasercConfig.mustRunAfter(prepareConfig) + firebaseProjectCreate.mustRunAfter(prepareFirebasercConfig) + firebaseWebAppCreate.mustRunAfter(firebaseProjectCreate) + getSdkConfigWebApp.mustRunAfter(firebaseWebAppCreate) + prepareFirebaseOptionsDart.mustRunAfter(getSdkConfigWebApp) + flutterPubGetPG.mustRunAfter(prepareFirebaseOptionsDart) + flutterPubRunPG.mustRunAfter(flutterPubGetPG) + flutterPubGetTob.mustRunAfter(flutterPubRunPG) + flutterPubRunTob.mustRunAfter(flutterPubGetTob) + flutterBuildWeb.mustRunAfter(flutterPubRunTob) + firebaseDeploy.mustRunAfter(flutterBuildWeb) +} diff --git a/learning/tour-of-beam/terraform/cloud_functions/main.tf b/learning/tour-of-beam/terraform/cloud_functions/main.tf new file mode 100644 index 000000000000..baa3813c5ce1 --- /dev/null +++ b/learning/tour-of-beam/terraform/cloud_functions/main.tf @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +resource "google_cloudfunctions_function" "cloud_function" { + count = length(var.entry_point_names) + name = "${var.environment}_${var.entry_point_names[count.index]}" + runtime = "go116" + available_memory_mb = 128 + project = var.project_id + service_account_email = var.service_account_id + source_archive_bucket = var.source_archive_bucket + source_archive_object = var.source_archive_object + region = var.region + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered + entry_point = var.entry_point_names[count.index] + + environment_variables = { + DATASTORE_PROJECT_ID=var.project_id + GOOGLE_PROJECT_ID=var.project_id + PLAYGROUND_ROUTER_HOST=var.pg_router_host + } + + timeouts { + create = "20m" + delete = "20m" + } + +} + +# Create IAM entry so all users can invoke the function +resource "google_cloudfunctions_function_iam_member" "invoker" { + count = length(google_cloudfunctions_function.cloud_function) + project = google_cloudfunctions_function.cloud_function[count.index].project + region = google_cloudfunctions_function.cloud_function[count.index].region + cloud_function = google_cloudfunctions_function.cloud_function[count.index].name + + role = "roles/cloudfunctions.invoker" + member = "allUsers" + + depends_on = [google_cloudfunctions_function.cloud_function] +} diff --git a/learning/tour-of-beam/terraform/cloud_functions/output.tf b/learning/tour-of-beam/terraform/cloud_functions/output.tf new file mode 100644 index 000000000000..a8ba150ed5f9 --- /dev/null +++ b/learning/tour-of-beam/terraform/cloud_functions/output.tf @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +output "cloud-function-trigger-url" { + value = google_cloudfunctions_function.cloud_function.*.https_trigger_url +} diff --git a/learning/tour-of-beam/terraform/cloud_functions/variables.tf b/learning/tour-of-beam/terraform/cloud_functions/variables.tf new file mode 100644 index 000000000000..3a3ab42d0540 --- /dev/null +++ b/learning/tour-of-beam/terraform/cloud_functions/variables.tf @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "service_account_id" { + description = "The name of Service Account to run Cloud Function" +} + +variable "project_id" { + description = "The GCP Project ID of function" +} + +variable "region" { + description = "The GCP Region of function" +} + +variable "source_archive_bucket" { + description = "The GCS bucket containing the zip archive which contains the function" +} +variable "source_archive_object" { + description = "The source archive object (file) in archive bucket" +} + +variable "entry_point_names" { + type = list + default = ["getSdkList", "getContentTree", "getUnitContent", "getUserProgress", "postUnitComplete", "postUserCode", "postDeleteProgress"] +} + +variable "pg_router_host" { + description = "Hostname:port of Playground GKE cluster's router grpc workload" +} + +variable "environment" {} diff --git a/learning/tour-of-beam/terraform/environment/test/state.tfbackend b/learning/tour-of-beam/terraform/environment/test/state.tfbackend new file mode 100644 index 000000000000..93e7554cf8cb --- /dev/null +++ b/learning/tour-of-beam/terraform/environment/test/state.tfbackend @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +bucket = "tfstate-bucket-tob-1" diff --git a/learning/tour-of-beam/terraform/functions_buckets/main.tf b/learning/tour-of-beam/terraform/functions_buckets/main.tf new file mode 100644 index 000000000000..41fbffbbd2c0 --- /dev/null +++ b/learning/tour-of-beam/terraform/functions_buckets/main.tf @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +resource "google_storage_bucket" "cloud_functions_bucket" { + name = var.cloudfunctions_bucket + location = var.region + storage_class = "STANDARD" +} + +resource "google_storage_bucket_object" "zip" { + # Use an MD5 here. If there's no changes to the source code, this won't change either. + # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing + # a redeployment when it has! + name = "${data.archive_file.source_code.output_md5}.zip" + bucket = google_storage_bucket.cloud_functions_bucket.name + source = data.archive_file.source_code.output_path + content_type = "application/zip" +} diff --git a/learning/tour-of-beam/terraform/functions_buckets/output.tf b/learning/tour-of-beam/terraform/functions_buckets/output.tf new file mode 100644 index 000000000000..e2f16108f54a --- /dev/null +++ b/learning/tour-of-beam/terraform/functions_buckets/output.tf @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +output "functions-bucket-id" { + value = google_storage_bucket.cloud_functions_bucket.id +} + +output "functions-bucket-name" { + value = google_storage_bucket.cloud_functions_bucket.name +} + +output "function-bucket-object" { + value = google_storage_bucket_object.zip.name +} diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf b/learning/tour-of-beam/terraform/functions_buckets/variables.tf new file mode 100644 index 000000000000..126f66e609e6 --- /dev/null +++ b/learning/tour-of-beam/terraform/functions_buckets/variables.tf @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#Generates archive of source code +variable "cloudfunctions_bucket" { + description = "The name of the bucket to store cloud functions' source code" +} + +variable "region" { + description = "The GCS region" +} + +data "archive_file" "source_code" { + type = "zip" + source_dir = "../backend" + output_path = "/tmp/backend.zip" +} diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf new file mode 100644 index 000000000000..04880691a286 --- /dev/null +++ b/learning/tour-of-beam/terraform/main.tf @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module "setup" { + source = "./setup" + project_id = var.project_id + service_account_id = var.service_account_id + gcloud_init_account = var.gcloud_init_account + depends_on = [module.api_enable] +} + +module "functions_buckets" { + source = "./functions_buckets" + region = var.region + cloudfunctions_bucket = var.cloudfunctions_bucket + depends_on = [module.setup, module.api_enable] +} + +module "api_enable" { + source = "./api_enable" + project_id = var.project_id +} + +module "cloud_functions" { + source = "./cloud_functions" + region = var.region + project_id = var.project_id + pg_router_host = var.pg_router_host + environment = var.environment + service_account_id = module.setup.service-account-email + source_archive_bucket = module.functions_buckets.functions-bucket-name + source_archive_object = module.functions_buckets.function-bucket-object + depends_on = [module.functions_buckets, module.setup, module.api_enable] +} diff --git a/learning/tour-of-beam/terraform/output.tf b/learning/tour-of-beam/terraform/output.tf new file mode 100644 index 000000000000..f85f61497df3 --- /dev/null +++ b/learning/tour-of-beam/terraform/output.tf @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +output "service-account-email" { + value = module.setup.service-account-email +} + +output "cloud-function-trigger-url" { + value = module.cloud_functions.cloud-function-trigger-url +} + +output "functions-bucket-name" { + value = module.functions_buckets.functions-bucket-name +} + +output "function-bucket-object" { + value = module.functions_buckets.function-bucket-object +} diff --git a/learning/tour-of-beam/terraform/provider.tf b/learning/tour-of-beam/terraform/provider.tf new file mode 100644 index 000000000000..df421ee1fad4 --- /dev/null +++ b/learning/tour-of-beam/terraform/provider.tf @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +terraform { + backend "gcs" { + } + + required_providers { + google = { + source = "hashicorp/google" + version = "4.4.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/learning/tour-of-beam/terraform/setup/data.tf b/learning/tour-of-beam/terraform/setup/data.tf new file mode 100644 index 000000000000..6963c11b76ac --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/data.tf @@ -0,0 +1,3 @@ +data "external" "gcloud_account" { + program = ["gcloud", "config", "get-value", "core/account"] +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/iam.tf b/learning/tour-of-beam/terraform/setup/iam.tf new file mode 100644 index 000000000000..842f16380658 --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/iam.tf @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +resource "google_service_account" "cloud_function_sa" { + account_id = var.service_account_id + display_name = "Service Account to run Cloud Functions" +} + +resource "google_project_iam_member" "terraform_service_account_roles" { + for_each = toset([ + "roles/cloudfunctions.admin", "roles/storage.objectViewer", + "roles/iam.serviceAccountUser", "roles/datastore.user", + "roles/firebaseauth.viewer" + ]) + role = each.key + member = "serviceAccount:${google_service_account.cloud_function_sa.email}" + project = var.project_id +} + +resource "google_project_iam_member" "gcloud_user_required_roles" { + for_each = toset([ + "roles/cloudfunctions.admin", "roles/firebase.admin" + ]) + role = each.key + member = "user:${var.gcloud_init_account}" + project = var.project_id +} diff --git a/learning/tour-of-beam/terraform/setup/output.tf b/learning/tour-of-beam/terraform/setup/output.tf new file mode 100644 index 000000000000..cc3d34791868 --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/output.tf @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +output "service-account-email" { + value = google_service_account.cloud_function_sa.email +} diff --git a/learning/tour-of-beam/terraform/setup/variables.tf b/learning/tour-of-beam/terraform/setup/variables.tf new file mode 100644 index 000000000000..39898e69002e --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/variables.tf @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +variable "service_account_id" { + description = "The name of Service Account to run Cloud Function" +} + +variable "gcloud_init_account" { + description = "User Account ID logged in with gcloud init command (e.g. username@domain.com)" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf new file mode 100644 index 000000000000..87d2b8c07236 --- /dev/null +++ b/learning/tour-of-beam/terraform/variables.tf @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "cloudfunctions_bucket" { + description = "The bucket name that will store functions' source code" +} + +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +variable "service_account_id" { + description = "The ID of the service account responsible for running Google Cloud functions" + default = "tourofbeam-cloudfunction-sa" +} + +variable "region" { + description = "The region of the Google Cloud project within which resources are provisioned" +} + +variable "gcloud_init_account" { + description = "User Account ID logged in with gcloud init command (e.g. username@domain.com)" +} + +variable "pg_router_host" {} + +variable "environment" {} From 0ae72507851c2d1dd93723024d145b8459625a08 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 10 Mar 2023 17:42:05 +0500 Subject: [PATCH 02/35] ToB Frontend related updates --- learning/tour-of-beam/frontend/firebase.json | 16 ++++++++++++++++ .../frontend/lib/pages/tour/state.dart | 8 ++++---- .../client/cloud_functions_client.dart | 10 +++++----- learning/tour-of-beam/frontend/pubspec.yaml | 2 +- 4 files changed, 26 insertions(+), 10 deletions(-) create mode 100644 learning/tour-of-beam/frontend/firebase.json diff --git a/learning/tour-of-beam/frontend/firebase.json b/learning/tour-of-beam/frontend/firebase.json new file mode 100644 index 000000000000..66037326c171 --- /dev/null +++ b/learning/tour-of-beam/frontend/firebase.json @@ -0,0 +1,16 @@ +{ + "hosting": { + "public": "build/web", + "ignore": [ + "firebase.json", + "**/.*", + "**/node_modules/**" + ], + "rewrites": [ + { + "source": "**", + "destination": "/index.html" + } + ] + } +} diff --git a/learning/tour-of-beam/frontend/lib/pages/tour/state.dart b/learning/tour-of-beam/frontend/lib/pages/tour/state.dart index 19646be75431..b49a1310be09 100644 --- a/learning/tour-of-beam/frontend/lib/pages/tour/state.dart +++ b/learning/tour-of-beam/frontend/lib/pages/tour/state.dart @@ -213,12 +213,12 @@ class TourNotifier extends ChangeNotifier with PageStateMixin { return playgroundController; } - @override - void dispose() { +@override + Future dispose() async { _unitContentCache.removeListener(_onUnitChanged); contentTreeController.removeListener(_onUnitChanged); _appNotifier.removeListener(_onAppNotifierChanged); _authNotifier.removeListener(_onUnitProgressChanged); - super.dispose(); - } + await super.dispose(); + } } diff --git a/learning/tour-of-beam/frontend/lib/repositories/client/cloud_functions_client.dart b/learning/tour-of-beam/frontend/lib/repositories/client/cloud_functions_client.dart index 8986de435290..5a1840bcddb1 100644 --- a/learning/tour-of-beam/frontend/lib/repositories/client/cloud_functions_client.dart +++ b/learning/tour-of-beam/frontend/lib/repositories/client/cloud_functions_client.dart @@ -37,7 +37,7 @@ class CloudFunctionsTobClient extends TobClient { Future getSdks() async { final json = await http.get( Uri.parse( - '$cloudFunctionsBaseUrl/getSdkList', + '${cloudFunctionsBaseUrl}getSdkList', ), ); @@ -49,7 +49,7 @@ class CloudFunctionsTobClient extends TobClient { Future getContentTree(String sdkId) async { final json = await http.get( Uri.parse( - '$cloudFunctionsBaseUrl/getContentTree?sdk=$sdkId', + '${cloudFunctionsBaseUrl}getContentTree?sdk=$sdkId', ), ); @@ -62,7 +62,7 @@ class CloudFunctionsTobClient extends TobClient { Future getUnitContent(String sdkId, String unitId) async { final json = await http.get( Uri.parse( - '$cloudFunctionsBaseUrl/getUnitContent?sdk=$sdkId&id=$unitId', + '${cloudFunctionsBaseUrl}getUnitContent?sdk=$sdkId&id=$unitId', ), ); @@ -78,7 +78,7 @@ class CloudFunctionsTobClient extends TobClient { } final json = await http.get( Uri.parse( - '$cloudFunctionsBaseUrl/getUserProgress?sdk=$sdkId', + '${cloudFunctionsBaseUrl}getUserProgress?sdk=$sdkId', ), headers: { HttpHeaders.authorizationHeader: 'Bearer $token', @@ -94,7 +94,7 @@ class CloudFunctionsTobClient extends TobClient { final token = await GetIt.instance.get().getToken(); await http.post( Uri.parse( - '$cloudFunctionsBaseUrl/postUnitComplete?sdk=$sdkId&id=$id', + '${cloudFunctionsBaseUrl}postUnitComplete?sdk=$sdkId&id=$id', ), headers: { HttpHeaders.authorizationHeader: 'Bearer $token', diff --git a/learning/tour-of-beam/frontend/pubspec.yaml b/learning/tour-of-beam/frontend/pubspec.yaml index be0579d0885d..dc2acb404b5f 100644 --- a/learning/tour-of-beam/frontend/pubspec.yaml +++ b/learning/tour-of-beam/frontend/pubspec.yaml @@ -23,7 +23,7 @@ publish_to: 'none' version: 0.1.0 environment: - sdk: '>=2.18.1 <3.0.0' + sdk: '>=2.19.2 <4.0.0' flutter: '>=3.3.2' dependencies: From f10c10e942166603096520fc7f3c2dc1e786a36f Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 10 Mar 2023 17:43:11 +0500 Subject: [PATCH 03/35] Update settings.gradle.kts --- settings.gradle.kts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/settings.gradle.kts b/settings.gradle.kts index b2f8c89680d1..7a78baaea856 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -82,6 +82,9 @@ include(":runners:core-construction-java") include(":runners:core-java") include(":runners:direct-java") include(":runners:extensions-java:metrics") +include(":learning") +include(":learning:tour-of-beam") +include(":learning:tour-of-beam:terraform") /* Begin Flink Runner related settings */ // Flink 1.12 include(":runners:flink:1.12") From 8d9fe7795c674e7cc559bbee7aba10d33ece1d1d Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Mon, 13 Mar 2023 21:08:29 +0500 Subject: [PATCH 04/35] Deleted redundant file and minor README change --- learning/tour-of-beam/terraform/README.md | 2 +- learning/tour-of-beam/terraform/setup/data.tf | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 learning/tour-of-beam/terraform/setup/data.tf diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 3b29bf072fe4..ac349e802631 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -85,7 +85,7 @@ gcloud container clusters get-credentials --region `chosen_gke_zone` `gke_name` 5. Run the following command from the top-level repository folder ("beam") to deploy the Tour of Beam Backend infrastructure: ``` -./gradlew learning:tour-of-beam:terraform:InitBackend -Pproject_environment="environment_name" -Pproject_id="gcp-project-id" -Pgcloud_account=`gcloud config get-value core/account` +./gradlew learning:tour-of-beam:terraform:InitBackend -Pgcloud_account=`gcloud config get-value core/account` -Pproject_environment="environment_name" -Pproject_id="gcp-project-id" ``` Where: - **project_environment** - environment name diff --git a/learning/tour-of-beam/terraform/setup/data.tf b/learning/tour-of-beam/terraform/setup/data.tf deleted file mode 100644 index 6963c11b76ac..000000000000 --- a/learning/tour-of-beam/terraform/setup/data.tf +++ /dev/null @@ -1,3 +0,0 @@ -data "external" "gcloud_account" { - program = ["gcloud", "config", "get-value", "core/account"] -} \ No newline at end of file From 289c07afc1021d45a09e8979a4339e8a019e808a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 16 Mar 2023 22:07:18 +0500 Subject: [PATCH 05/35] Addressing comments in the PR --- learning/tour-of-beam/terraform/cloud_functions/main.tf | 4 ++-- .../test/state.tfbackend => functions_buckets/data.tf} | 8 +++++--- .../tour-of-beam/terraform/functions_buckets/variables.tf | 6 ------ 3 files changed, 7 insertions(+), 11 deletions(-) rename learning/tour-of-beam/terraform/{environment/test/state.tfbackend => functions_buckets/data.tf} (86%) diff --git a/learning/tour-of-beam/terraform/cloud_functions/main.tf b/learning/tour-of-beam/terraform/cloud_functions/main.tf index baa3813c5ce1..1afac61f431e 100644 --- a/learning/tour-of-beam/terraform/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform/cloud_functions/main.tf @@ -48,8 +48,8 @@ resource "google_cloudfunctions_function" "cloud_function" { # Create IAM entry so all users can invoke the function resource "google_cloudfunctions_function_iam_member" "invoker" { count = length(google_cloudfunctions_function.cloud_function) - project = google_cloudfunctions_function.cloud_function[count.index].project - region = google_cloudfunctions_function.cloud_function[count.index].region + project = var.project_id + region = var.region cloud_function = google_cloudfunctions_function.cloud_function[count.index].name role = "roles/cloudfunctions.invoker" diff --git a/learning/tour-of-beam/terraform/environment/test/state.tfbackend b/learning/tour-of-beam/terraform/functions_buckets/data.tf similarity index 86% rename from learning/tour-of-beam/terraform/environment/test/state.tfbackend rename to learning/tour-of-beam/terraform/functions_buckets/data.tf index 93e7554cf8cb..0bd83744ba73 100644 --- a/learning/tour-of-beam/terraform/environment/test/state.tfbackend +++ b/learning/tour-of-beam/terraform/functions_buckets/data.tf @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,6 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -bucket = "tfstate-bucket-tob-1" +data "archive_file" "source_code" { + type = "zip" + source_dir = "../backend" + output_path = "/tmp/backend.zip" +} diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf b/learning/tour-of-beam/terraform/functions_buckets/variables.tf index 126f66e609e6..dce96e2bbe54 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/variables.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/variables.tf @@ -23,9 +23,3 @@ variable "cloudfunctions_bucket" { variable "region" { description = "The GCS region" } - -data "archive_file" "source_code" { - type = "zip" - source_dir = "../backend" - output_path = "/tmp/backend.zip" -} From 8e3f7a1d3b2fbe01ff0d3ba325b65a947a7eb364 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 16 Mar 2023 22:10:34 +0500 Subject: [PATCH 06/35] Added newline at the end of variables.tf file --- learning/tour-of-beam/terraform/setup/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform/setup/variables.tf b/learning/tour-of-beam/terraform/setup/variables.tf index 39898e69002e..1de0ec95949b 100644 --- a/learning/tour-of-beam/terraform/setup/variables.tf +++ b/learning/tour-of-beam/terraform/setup/variables.tf @@ -25,4 +25,4 @@ variable "service_account_id" { variable "gcloud_init_account" { description = "User Account ID logged in with gcloud init command (e.g. username@domain.com)" -} \ No newline at end of file +} From 4f72d86ea11a687310c9ff8b69636d2c4cd2d2b2 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 29 Mar 2023 21:18:40 +0500 Subject: [PATCH 07/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index ac349e802631..9670101bd53d 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -81,7 +81,7 @@ gcloud auth application-default login gcloud container clusters get-credentials --region `chosen_gke_zone` `gke_name` --project `project_id` ``` -# Deploy the Tour of Beam Backend: +# Deploy the Tour of Beam Backend Infrastructure: 5. Run the following command from the top-level repository folder ("beam") to deploy the Tour of Beam Backend infrastructure: ``` @@ -91,7 +91,7 @@ Where: - **project_environment** - environment name - **project_id** - name of your GCP Project ID -# Deploy the Tour of Beam Frontend: +# Deploy the Tour of Beam Frontend Infrastructure: 6. Run the following command and follow the instructions to configure authentication for Firebase: ``` From 7141c51bd75ca13aae02baab1a1f496d2c636405 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Apr 2023 20:46:25 +0500 Subject: [PATCH 08/35] Updates related to Tour of Beam infrastructure --- learning/tour-of-beam/terraform/README.md | 14 ++--- .../tour-of-beam/terraform/api_enable/main.tf | 1 - .../tour-of-beam/terraform/build.gradle.kts | 54 ++++++------------- .../terraform/cloud_functions/variables.tf | 29 +++++++++- .../terraform/functions_buckets/data.tf | 1 + .../output.tf => functions_buckets/locals.tf} | 8 +-- .../terraform/functions_buckets/main.tf | 4 +- .../terraform/functions_buckets/variables.tf | 16 +++++- learning/tour-of-beam/terraform/main.tf | 1 - .../variables.tf => setup/locals.tf} | 7 +-- .../tour-of-beam/terraform/setup/output.tf | 1 + .../tour-of-beam/terraform/setup/variables.tf | 18 +++++++ learning/tour-of-beam/terraform/variables.tf | 9 ++-- 13 files changed, 101 insertions(+), 62 deletions(-) rename learning/tour-of-beam/terraform/{cloud_functions/output.tf => functions_buckets/locals.tf} (80%) rename learning/tour-of-beam/terraform/{api_enable/variables.tf => setup/locals.tf} (81%) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 9670101bd53d..f43f56017b9b 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -37,7 +37,6 @@ This guide provides instructions on how to deploy the Tour of Beam environment o 4. An OS with the following software installed: * [Java](https://adoptopenjdk.net/) -* [NodeJS & npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm/) * [Flutter (3.7.3 >)](https://docs.flutter.dev/get-started/install) * [Dart SDK (2.19.2)](https://dart.dev/get-dart) * [Firebase-tools CLI](https://www.npmjs.com/package/firebase-tools) @@ -46,21 +45,22 @@ This guide provides instructions on how to deploy the Tour of Beam environment o * [Kubectl authentication plugin](https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke) * [Go](https://go.dev/doc/install) -5. Apache Beam Git repository cloned locally +5. Beam Playground environment (Existing GKE Cluster will be required particularly) + +6. Apache Beam Git repository cloned locally # Prepare deployment configuration: -Tour of Beam backend uses `terraform.tfvars` located in `learning/tour-of-beam/terraform/environment/environment_name/` to define variables specific to an environment (e.g., prod, test, staging). Follow the steps below to prepare the deployment configuration:
-1. Create a folder (referred to as `environment_name`) to define a new environment and place configuration files into it: + `common.tfvars` located in `learning/tour-of-beam/terraform` to define variables specific to an environment (e.g., prod, test, staging). Follow the steps below to prepare the deployment configuration:
+1. Create a `common.tfvars` and `state.tfbackend` files in terraform directory `learning/tour-of-beam/terraform`: -* `terraform.tfvars` environment variables: +* Populate `common.tfvars` with next variables: ``` project_id = "gcp_project_id" # Your GCP Project ID -cloudfunctions_bucket = "gcs_bucket_name" # Globally unique name of the bucket to store cloud functions' source code region = "gcp_region" # Your GCP resources region service_account_id = "service_account_name" # Name of GCP service account to run Tour of Beam cloud functions ``` -* `state.tfbackend` environment variables: +* `state.tfbackend` with: ``` bucket = "bucket_name" # Your created bucket name for terraform tfstate file ``` diff --git a/learning/tour-of-beam/terraform/api_enable/main.tf b/learning/tour-of-beam/terraform/api_enable/main.tf index 3a61a090fe9e..9d792c246371 100644 --- a/learning/tour-of-beam/terraform/api_enable/main.tf +++ b/learning/tour-of-beam/terraform/api_enable/main.tf @@ -16,7 +16,6 @@ # under the License. resource "google_project_service" "required_services" { - project = var.project_id for_each = toset([ "cloudresourcemanager", "iam", diff --git a/learning/tour-of-beam/terraform/build.gradle.kts b/learning/tour-of-beam/terraform/build.gradle.kts index 4406861d4ed4..ed4ccc7b1094 100644 --- a/learning/tour-of-beam/terraform/build.gradle.kts +++ b/learning/tour-of-beam/terraform/build.gradle.kts @@ -25,79 +25,55 @@ plugins { } terraformPlugin { - terraformVersion.set("1.0.9") + terraformVersion.set("1.4.2") } /* init Infrastructure for migrate */ tasks.register("terraformInit") { - // exec args can be passed by commandline, for example - var environment = project.property("project_environment") as String - args( - "init", "-migrate-state", - "-backend-config=./environment/$environment/state.tfbackend", - "-var=environment=$environment", - if (file("./environment/$environment/terraform.tfvars").exists()) { - "-var-file=./environment/$environment/terraform.tfvars" - } else { - "-no-color" - } - ) - } + // exec args can be passed by commandline, for example + args( + "init", "-migrate-state", + "-backend-config=./state.tfbackend" + ) +} /* refresh Infrastucture for remote state */ tasks.register("terraformRef") { - var environment = project.property("project_environment") as String args( "refresh", "-lock=false", - "-var=environment=$environment", - if (file("./environment/$environment/terraform.tfvars").exists()) { - "-var-file=./environment/$environment/terraform.tfvars" - } else { - "-no-color" - } + "-var-file=./common.tfvars" ) } tasks.register("terraformApplyBackend") { group = "backend-deploy" var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String - var environment = project.property("project_environment") as String - var gcloud_account = project.property("gcloud_account") as String args( "apply", "-auto-approve", "-lock=false", "-parallelism=3", "-var=pg_router_host=$pg_router_host", - "-var=gcloud_init_account=$gcloud_account", - "-var=environment=$environment", - if (file("./environment/$environment/terraform.tfvars").exists()) { - "-var-file=./environment/$environment/terraform.tfvars" - } else { - "-no-color" - } + "-var=gcloud_init_account=$(gcloud config get-value core/account)", + "-var=project_id=$(gcloud config get-value project)", + "-var-file=./common.tfvars" ) + tasks.getByName("uploadLearningMaterials").mustRunAfter(this) } tasks.register("terraformDestroy") { var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String - var environment = project.property("project_environment") as String - var gcloud_account = project.property("gcloud_account") as String args( "destroy", "-auto-approve", "-lock=false", "-var=pg_router_host=$pg_router_host", - "-var=environment=$environment", - "-var=gcloud_init_account=$gcloud_account", - if (file("./environment/$environment/terraform.tfvars").exists()) { - "-var-file=./environment/$environment/terraform.tfvars" - } else { - "-no-color" - } + "-var=gcloud_init_account=$(gcloud config get-value core/account)", + "-var=project_id=$(gcloud config get-value project)", + "-var-file=./common.tfvars" ) } diff --git a/learning/tour-of-beam/terraform/cloud_functions/variables.tf b/learning/tour-of-beam/terraform/cloud_functions/variables.tf index 3a3ab42d0540..0139a5c2a85c 100644 --- a/learning/tour-of-beam/terraform/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform/cloud_functions/variables.tf @@ -15,32 +15,57 @@ # specific language governing permissions and limitations # under the License. +# This will be used to generate certain variables values +resource "random_string" "id" { + length = 4 + upper = false + special = false +} + +# This will be used to generate certain variables +variable "resource_name_prefix" { + type = string + description = "The resource name prefix applied to all resource naming for the application" + default = "tour-of-beam" +} + +# Taken from output of SETUP module variable "service_account_id" { description = "The name of Service Account to run Cloud Function" } +# Required for environment variables inside of cloud functions +# Generated by command (gcloud config get-value project) in Kotlin Gradle script variable "project_id" { description = "The GCP Project ID of function" } variable "region" { - description = "The GCP Region of function" + description = "The GCP Region where cloud functions will be created" } +# Taken from output of FUNCTIONS_BUCKETS module variable "source_archive_bucket" { description = "The GCS bucket containing the zip archive which contains the function" } + +# Taken from output of FUNCTIONS_BUCKETS module variable "source_archive_object" { description = "The source archive object (file) in archive bucket" } +# Constant. Will be served as cloud functions URLs/endpoints variable "entry_point_names" { type = list default = ["getSdkList", "getContentTree", "getUnitContent", "getUserProgress", "postUnitComplete", "postUserCode", "postDeleteProgress"] } +# Generated during execution Kotlin Gradle script variable "pg_router_host" { description = "Hostname:port of Playground GKE cluster's router grpc workload" } -variable "environment" {} +# To be provided manually as argument for Kotlin Gradle script +variable "environment" { + description = "The name of the environment for deployment of cloud functions. Will be appended to the name of cloud functions" +} diff --git a/learning/tour-of-beam/terraform/functions_buckets/data.tf b/learning/tour-of-beam/terraform/functions_buckets/data.tf index 0bd83744ba73..952eaa265543 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/data.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/data.tf @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +# Data resource to archive source code for cloud functions data "archive_file" "source_code" { type = "zip" source_dir = "../backend" diff --git a/learning/tour-of-beam/terraform/cloud_functions/output.tf b/learning/tour-of-beam/terraform/functions_buckets/locals.tf similarity index 80% rename from learning/tour-of-beam/terraform/cloud_functions/output.tf rename to learning/tour-of-beam/terraform/functions_buckets/locals.tf index a8ba150ed5f9..cb20db08d6f5 100644 --- a/learning/tour-of-beam/terraform/cloud_functions/output.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/locals.tf @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. -output "cloud-function-trigger-url" { - value = google_cloudfunctions_function.cloud_function.*.https_trigger_url -} +# Local value to store generated GCS bucket name for source code (Cloud Functions) + +locals { + cloudfunctions_bucket = "${var.resource_name_prefix}-cfstorage-${random_string.id.result}" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/functions_buckets/main.tf b/learning/tour-of-beam/terraform/functions_buckets/main.tf index 41fbffbbd2c0..a6fcfb7aa430 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/main.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/main.tf @@ -15,16 +15,18 @@ # specific language governing permissions and limitations # under the License. +# This creates GCS bucket for source code for cloud functions resource "google_storage_bucket" "cloud_functions_bucket" { name = var.cloudfunctions_bucket location = var.region storage_class = "STANDARD" } +# This creates object for a bucket with the source code resource "google_storage_bucket_object" "zip" { # Use an MD5 here. If there's no changes to the source code, this won't change either. # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing - # a redeployment when it has! + # a redeployment when it has name = "${data.archive_file.source_code.output_md5}.zip" bucket = google_storage_bucket.cloud_functions_bucket.name source = data.archive_file.source_code.output_path diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf b/learning/tour-of-beam/terraform/functions_buckets/variables.tf index dce96e2bbe54..f4433c8323c7 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/variables.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/variables.tf @@ -15,11 +15,23 @@ # specific language governing permissions and limitations # under the License. -#Generates archive of source code +resource "random_string" "id" { + length = 4 + upper = false + special = false +} + +variable "resource_name_prefix" { + type = string + description = "The resource name prefix applied to all resource naming for the application" + default = "tour-of-beam" +} + variable "cloudfunctions_bucket" { description = "The name of the bucket to store cloud functions' source code" + default = local.cloudfunctions_bucket } variable "region" { - description = "The GCS region" + description = "The GCP region where GCS bucket will be created (For Cloud Functions source code)" } diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf index 04880691a286..90c0e04ec241 100644 --- a/learning/tour-of-beam/terraform/main.tf +++ b/learning/tour-of-beam/terraform/main.tf @@ -32,7 +32,6 @@ module "functions_buckets" { module "api_enable" { source = "./api_enable" - project_id = var.project_id } module "cloud_functions" { diff --git a/learning/tour-of-beam/terraform/api_enable/variables.tf b/learning/tour-of-beam/terraform/setup/locals.tf similarity index 81% rename from learning/tour-of-beam/terraform/api_enable/variables.tf rename to learning/tour-of-beam/terraform/setup/locals.tf index f4a375110500..c50fec9722ad 100644 --- a/learning/tour-of-beam/terraform/api_enable/variables.tf +++ b/learning/tour-of-beam/terraform/setup/locals.tf @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. -variable "project_id" { - description = "The ID of the Google Cloud project within which resources are provisioned" -} +# Local value to store generated Cloud Functions Service account name +locals { + cloudfunctions_service_account = "${var.resource_name_prefix}-sa-${random_string.id.result}" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/output.tf b/learning/tour-of-beam/terraform/setup/output.tf index cc3d34791868..7407e0814116 100644 --- a/learning/tour-of-beam/terraform/setup/output.tf +++ b/learning/tour-of-beam/terraform/setup/output.tf @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +# This output required to assign it to cloud functions output "service-account-email" { value = google_service_account.cloud_function_sa.email } diff --git a/learning/tour-of-beam/terraform/setup/variables.tf b/learning/tour-of-beam/terraform/setup/variables.tf index 1de0ec95949b..6cd4e09b218a 100644 --- a/learning/tour-of-beam/terraform/setup/variables.tf +++ b/learning/tour-of-beam/terraform/setup/variables.tf @@ -15,14 +15,32 @@ # specific language governing permissions and limitations # under the License. +resource "random_string" "id" { + length = 4 + upper = false + special = false +} + +variable "resource_name_prefix" { + type = string + description = "The resource name prefix applied to all resource naming for the application" + default = "tour-of-beam" +} + +# Required and not inferred from the provider argument. +# Generated by command (gcloud config get-value project) in Kotlin Gradle script variable "project_id" { description = "The ID of the Google Cloud project within which resources are provisioned" } +# This service account ID is generated using prefix, constant string and random digits variable "service_account_id" { description = "The name of Service Account to run Cloud Function" + default = local.cloudfunctions_service_account + } +# This variable is generated by command (gcloud config get-value core/account) in Kotlin Gradle script variable "gcloud_init_account" { description = "User Account ID logged in with gcloud init command (e.g. username@domain.com)" } diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf index 87d2b8c07236..afa2581289cc 100644 --- a/learning/tour-of-beam/terraform/variables.tf +++ b/learning/tour-of-beam/terraform/variables.tf @@ -25,7 +25,6 @@ variable "project_id" { variable "service_account_id" { description = "The ID of the service account responsible for running Google Cloud functions" - default = "tourofbeam-cloudfunction-sa" } variable "region" { @@ -36,6 +35,10 @@ variable "gcloud_init_account" { description = "User Account ID logged in with gcloud init command (e.g. username@domain.com)" } -variable "pg_router_host" {} +variable "pg_router_host" { + description = "Hostname:port of Playground GKE cluster's router grpc workload" +} -variable "environment" {} +variable "environment" { + description = "The name of the environment for deployment. Will create directory where terraform config files will be stored" +} From ef7ea84b25a4ca5fefe06f11bd491518d6c70a13 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Apr 2023 21:04:22 +0500 Subject: [PATCH 09/35] Update locals.tf --- learning/tour-of-beam/terraform/setup/locals.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform/setup/locals.tf b/learning/tour-of-beam/terraform/setup/locals.tf index c50fec9722ad..fe25ed28af5f 100644 --- a/learning/tour-of-beam/terraform/setup/locals.tf +++ b/learning/tour-of-beam/terraform/setup/locals.tf @@ -17,5 +17,5 @@ # Local value to store generated Cloud Functions Service account name locals { - cloudfunctions_service_account = "${var.resource_name_prefix}-sa-${random_string.id.result}" + cloudfunctions_service_account = "${var.resource_name_prefix}-cf-sa-${random_string.id.result}" } \ No newline at end of file From 8ccfc4431fb40a2917218eb712e4b993c55105ad Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Apr 2023 21:07:14 +0500 Subject: [PATCH 10/35] Output.tf updates --- learning/tour-of-beam/terraform/functions_buckets/output.tf | 4 ---- learning/tour-of-beam/terraform/setup/output.tf | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/learning/tour-of-beam/terraform/functions_buckets/output.tf b/learning/tour-of-beam/terraform/functions_buckets/output.tf index e2f16108f54a..ad7d0aa0b9b6 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/output.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/output.tf @@ -15,10 +15,6 @@ # specific language governing permissions and limitations # under the License. -output "functions-bucket-id" { - value = google_storage_bucket.cloud_functions_bucket.id -} - output "functions-bucket-name" { value = google_storage_bucket.cloud_functions_bucket.name } diff --git a/learning/tour-of-beam/terraform/setup/output.tf b/learning/tour-of-beam/terraform/setup/output.tf index 7407e0814116..f072ba059988 100644 --- a/learning/tour-of-beam/terraform/setup/output.tf +++ b/learning/tour-of-beam/terraform/setup/output.tf @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -# This output required to assign it to cloud functions +# This output required to assign service account to cloud functions output "service-account-email" { value = google_service_account.cloud_function_sa.email } From 3454a217b4aee3b1a1f553eecc95fab077c982db Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Apr 2023 21:10:33 +0500 Subject: [PATCH 11/35] Update output.tf --- learning/tour-of-beam/terraform/output.tf | 7 ------- 1 file changed, 7 deletions(-) diff --git a/learning/tour-of-beam/terraform/output.tf b/learning/tour-of-beam/terraform/output.tf index f85f61497df3..c9072eebed3e 100644 --- a/learning/tour-of-beam/terraform/output.tf +++ b/learning/tour-of-beam/terraform/output.tf @@ -15,13 +15,6 @@ # specific language governing permissions and limitations # under the License. -output "service-account-email" { - value = module.setup.service-account-email -} - -output "cloud-function-trigger-url" { - value = module.cloud_functions.cloud-function-trigger-url -} output "functions-bucket-name" { value = module.functions_buckets.functions-bucket-name From e92725e03bf49162f6509947e8af856062235621 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Apr 2023 21:52:44 +0500 Subject: [PATCH 12/35] Updates --- learning/tour-of-beam/terraform/README.md | 3 --- .../terraform/functions_buckets/locals.tf | 13 +++++++++- .../terraform/functions_buckets/output.tf | 4 +++ .../terraform/functions_buckets/variables.tf | 13 ---------- learning/tour-of-beam/terraform/main.tf | 2 +- learning/tour-of-beam/terraform/output.tf | 25 ------------------- .../tour-of-beam/terraform/setup/locals.tf | 13 ++++++++++ .../tour-of-beam/terraform/setup/output.tf | 4 +++ .../tour-of-beam/terraform/setup/variables.tf | 14 ----------- 9 files changed, 34 insertions(+), 57 deletions(-) delete mode 100644 learning/tour-of-beam/terraform/output.tf diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index f43f56017b9b..40b7d83e9da6 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -55,10 +55,7 @@ This guide provides instructions on how to deploy the Tour of Beam environment o * Populate `common.tfvars` with next variables: ``` -project_id = "gcp_project_id" # Your GCP Project ID region = "gcp_region" # Your GCP resources region -service_account_id = "service_account_name" # Name of GCP service account to run Tour of Beam cloud functions - ``` * `state.tfbackend` with: ``` diff --git a/learning/tour-of-beam/terraform/functions_buckets/locals.tf b/learning/tour-of-beam/terraform/functions_buckets/locals.tf index cb20db08d6f5..e13282ce4c1f 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/locals.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/locals.tf @@ -15,8 +15,19 @@ # specific language governing permissions and limitations # under the License. -# Local value to store generated GCS bucket name for source code (Cloud Functions) +resource "random_string" "id" { + length = 4 + upper = false + special = false +} + +variable "resource_name_prefix" { + type = string + description = "The resource name prefix applied to all resource naming for the application" + default = "tour-of-beam" +} +# Local value to store generated GCS bucket name for source code (Cloud Functions) locals { cloudfunctions_bucket = "${var.resource_name_prefix}-cfstorage-${random_string.id.result}" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/functions_buckets/output.tf b/learning/tour-of-beam/terraform/functions_buckets/output.tf index ad7d0aa0b9b6..653466fca27c 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/output.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/output.tf @@ -22,3 +22,7 @@ output "functions-bucket-name" { output "function-bucket-object" { value = google_storage_bucket_object.zip.name } + +output "cloudfunctions-bucket-name" { + value = local.cloudfunctions_bucket +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf b/learning/tour-of-beam/terraform/functions_buckets/variables.tf index f4433c8323c7..e15a3f25ac4d 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/variables.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/variables.tf @@ -15,21 +15,8 @@ # specific language governing permissions and limitations # under the License. -resource "random_string" "id" { - length = 4 - upper = false - special = false -} - -variable "resource_name_prefix" { - type = string - description = "The resource name prefix applied to all resource naming for the application" - default = "tour-of-beam" -} - variable "cloudfunctions_bucket" { description = "The name of the bucket to store cloud functions' source code" - default = local.cloudfunctions_bucket } variable "region" { diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf index 90c0e04ec241..93c14acc16ca 100644 --- a/learning/tour-of-beam/terraform/main.tf +++ b/learning/tour-of-beam/terraform/main.tf @@ -26,7 +26,7 @@ module "setup" { module "functions_buckets" { source = "./functions_buckets" region = var.region - cloudfunctions_bucket = var.cloudfunctions_bucket + cloudfunctions_bucket = module.functions_buckets.cloudfunctions-bucket-name depends_on = [module.setup, module.api_enable] } diff --git a/learning/tour-of-beam/terraform/output.tf b/learning/tour-of-beam/terraform/output.tf deleted file mode 100644 index c9072eebed3e..000000000000 --- a/learning/tour-of-beam/terraform/output.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -output "functions-bucket-name" { - value = module.functions_buckets.functions-bucket-name -} - -output "function-bucket-object" { - value = module.functions_buckets.function-bucket-object -} diff --git a/learning/tour-of-beam/terraform/setup/locals.tf b/learning/tour-of-beam/terraform/setup/locals.tf index fe25ed28af5f..5c0681c9fb14 100644 --- a/learning/tour-of-beam/terraform/setup/locals.tf +++ b/learning/tour-of-beam/terraform/setup/locals.tf @@ -16,6 +16,19 @@ # under the License. # Local value to store generated Cloud Functions Service account name + +resource "random_string" "id" { + length = 4 + upper = false + special = false +} + +variable "resource_name_prefix" { + type = string + description = "The resource name prefix applied to all resource naming for the application" + default = "tour-of-beam" +} + locals { cloudfunctions_service_account = "${var.resource_name_prefix}-cf-sa-${random_string.id.result}" } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/output.tf b/learning/tour-of-beam/terraform/setup/output.tf index f072ba059988..3adca0c186c8 100644 --- a/learning/tour-of-beam/terraform/setup/output.tf +++ b/learning/tour-of-beam/terraform/setup/output.tf @@ -19,3 +19,7 @@ output "service-account-email" { value = google_service_account.cloud_function_sa.email } + +output "cloudfunctions-service-account" { + value = local.cloudfunctions_service_account +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/variables.tf b/learning/tour-of-beam/terraform/setup/variables.tf index 6cd4e09b218a..3a136d6ff5ab 100644 --- a/learning/tour-of-beam/terraform/setup/variables.tf +++ b/learning/tour-of-beam/terraform/setup/variables.tf @@ -15,18 +15,6 @@ # specific language governing permissions and limitations # under the License. -resource "random_string" "id" { - length = 4 - upper = false - special = false -} - -variable "resource_name_prefix" { - type = string - description = "The resource name prefix applied to all resource naming for the application" - default = "tour-of-beam" -} - # Required and not inferred from the provider argument. # Generated by command (gcloud config get-value project) in Kotlin Gradle script variable "project_id" { @@ -36,8 +24,6 @@ variable "project_id" { # This service account ID is generated using prefix, constant string and random digits variable "service_account_id" { description = "The name of Service Account to run Cloud Function" - default = local.cloudfunctions_service_account - } # This variable is generated by command (gcloud config get-value core/account) in Kotlin Gradle script From a21f5d7bce0136f393b70b049226d10855624a58 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Apr 2023 22:37:43 +0500 Subject: [PATCH 13/35] Update main.tf --- learning/tour-of-beam/terraform/functions_buckets/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform/functions_buckets/main.tf b/learning/tour-of-beam/terraform/functions_buckets/main.tf index a6fcfb7aa430..d22dd5dd4a89 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/main.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/main.tf @@ -17,7 +17,7 @@ # This creates GCS bucket for source code for cloud functions resource "google_storage_bucket" "cloud_functions_bucket" { - name = var.cloudfunctions_bucket + name = local.cloudfunctions_bucket location = var.region storage_class = "STANDARD" } From 63b7cd44050ee8a476f20da8ff06bef66750135a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Apr 2023 22:40:00 +0500 Subject: [PATCH 14/35] Updates to cloudfunctions_bucket variable --- .../tour-of-beam/terraform/functions_buckets/variables.tf | 4 ---- learning/tour-of-beam/terraform/variables.tf | 4 ---- 2 files changed, 8 deletions(-) diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf b/learning/tour-of-beam/terraform/functions_buckets/variables.tf index e15a3f25ac4d..c71ea1f36615 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/variables.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/variables.tf @@ -15,10 +15,6 @@ # specific language governing permissions and limitations # under the License. -variable "cloudfunctions_bucket" { - description = "The name of the bucket to store cloud functions' source code" -} - variable "region" { description = "The GCP region where GCS bucket will be created (For Cloud Functions source code)" } diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf index afa2581289cc..651986c51489 100644 --- a/learning/tour-of-beam/terraform/variables.tf +++ b/learning/tour-of-beam/terraform/variables.tf @@ -15,10 +15,6 @@ # specific language governing permissions and limitations # under the License. -variable "cloudfunctions_bucket" { - description = "The bucket name that will store functions' source code" -} - variable "project_id" { description = "The ID of the Google Cloud project within which resources are provisioned" } From 274d210207c5c95e222a878f4ab357091c3b2b5d Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Apr 2023 23:08:37 +0500 Subject: [PATCH 15/35] service_account_id changes --- learning/tour-of-beam/terraform/setup/iam.tf | 2 +- learning/tour-of-beam/terraform/setup/variables.tf | 5 ----- learning/tour-of-beam/terraform/variables.tf | 4 ---- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/learning/tour-of-beam/terraform/setup/iam.tf b/learning/tour-of-beam/terraform/setup/iam.tf index 842f16380658..08ac5e8bacba 100644 --- a/learning/tour-of-beam/terraform/setup/iam.tf +++ b/learning/tour-of-beam/terraform/setup/iam.tf @@ -16,7 +16,7 @@ # under the License. resource "google_service_account" "cloud_function_sa" { - account_id = var.service_account_id + account_id = local.cloudfunctions_service_account display_name = "Service Account to run Cloud Functions" } diff --git a/learning/tour-of-beam/terraform/setup/variables.tf b/learning/tour-of-beam/terraform/setup/variables.tf index 3a136d6ff5ab..dcb6f54808b0 100644 --- a/learning/tour-of-beam/terraform/setup/variables.tf +++ b/learning/tour-of-beam/terraform/setup/variables.tf @@ -21,11 +21,6 @@ variable "project_id" { description = "The ID of the Google Cloud project within which resources are provisioned" } -# This service account ID is generated using prefix, constant string and random digits -variable "service_account_id" { - description = "The name of Service Account to run Cloud Function" -} - # This variable is generated by command (gcloud config get-value core/account) in Kotlin Gradle script variable "gcloud_init_account" { description = "User Account ID logged in with gcloud init command (e.g. username@domain.com)" diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf index 651986c51489..d9debeeba851 100644 --- a/learning/tour-of-beam/terraform/variables.tf +++ b/learning/tour-of-beam/terraform/variables.tf @@ -19,10 +19,6 @@ variable "project_id" { description = "The ID of the Google Cloud project within which resources are provisioned" } -variable "service_account_id" { - description = "The ID of the service account responsible for running Google Cloud functions" -} - variable "region" { description = "The region of the Google Cloud project within which resources are provisioned" } From 2238064fb981796a1ee12e37877c065ab54b0086 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Tue, 18 Apr 2023 23:11:10 +0500 Subject: [PATCH 16/35] Update main.tf --- learning/tour-of-beam/terraform/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf index 93c14acc16ca..75a3896835e3 100644 --- a/learning/tour-of-beam/terraform/main.tf +++ b/learning/tour-of-beam/terraform/main.tf @@ -18,7 +18,6 @@ module "setup" { source = "./setup" project_id = var.project_id - service_account_id = var.service_account_id gcloud_init_account = var.gcloud_init_account depends_on = [module.api_enable] } @@ -26,7 +25,6 @@ module "setup" { module "functions_buckets" { source = "./functions_buckets" region = var.region - cloudfunctions_bucket = module.functions_buckets.cloudfunctions-bucket-name depends_on = [module.setup, module.api_enable] } From 5248ee11ec8a24acda215288a244f1556e595af8 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 19 Apr 2023 18:26:55 +0500 Subject: [PATCH 17/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 37 +++++++++++++++-------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 40b7d83e9da6..da7c49ba35c6 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -50,17 +50,14 @@ This guide provides instructions on how to deploy the Tour of Beam environment o 6. Apache Beam Git repository cloned locally # Prepare deployment configuration: - `common.tfvars` located in `learning/tour-of-beam/terraform` to define variables specific to an environment (e.g., prod, test, staging). Follow the steps below to prepare the deployment configuration:
-1. Create a `common.tfvars` and `state.tfbackend` files in terraform directory `learning/tour-of-beam/terraform`: -* Populate `common.tfvars` with next variables: ``` -region = "gcp_region" # Your GCP resources region -``` -* `state.tfbackend` with: +1. Navigate to `beam/learning/tour-of-beam/terraform` + ``` -bucket = "bucket_name" # Your created bucket name for terraform tfstate file +cd beam/learning/tour-of-beam/terraform ``` + 2. Configure authentication for the Google Cloud Platform (GCP) ``` gcloud init @@ -73,24 +70,40 @@ gcloud auth application-default login ``` gcloud auth configure-docker `chosen_region`-docker.pkg.dev ``` + 4. And the authentication in GCP Google Kubernetes Engine: ``` gcloud container clusters get-credentials --region `chosen_gke_zone` `gke_name` --project `project_id` ``` +5. Create datastore indexes: +``` +gcloud datastore indexes create ../backend/internal/storage/index.yaml +``` + # Deploy the Tour of Beam Backend Infrastructure: -5. Run the following command from the top-level repository folder ("beam") to deploy the Tour of Beam Backend infrastructure: +6. Initialize terraform ``` -./gradlew learning:tour-of-beam:terraform:InitBackend -Pgcloud_account=`gcloud config get-value core/account` -Pproject_environment="environment_name" -Pproject_id="gcp-project-id" +terraform init -backend-config="bucket=`created_gcs_bucket`" ``` + +7. Run terraform apply to create Tour-Of-Beam backend infrastructure +``` +terraform plan -var gcloud_init_account=$(gcloud config get-value core/account) \ +-var environment="test" \ +-var region="us-east1" \ +-var project_id=$(gcloud config get-value project) \ +-var pg_router_host=$(kubectl get svc -l app=backend-router-grpc -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}') +``` + Where: -- **project_environment** - environment name -- **project_id** - name of your GCP Project ID +- **environment** - environment name +- **region** - GCP region for your infrastructure # Deploy the Tour of Beam Frontend Infrastructure: -6. Run the following command and follow the instructions to configure authentication for Firebase: +8. Run the following command and follow the instructions to configure authentication for Firebase: ``` firebase login --no-localhost ``` From 0017def1e5ee0261b706a59ea0dea77914ffb73a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 20 Apr 2023 00:29:20 +0500 Subject: [PATCH 18/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 155 ++++++++++++++++++++-- 1 file changed, 141 insertions(+), 14 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index da7c49ba35c6..a862b08a93b8 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -51,7 +51,7 @@ This guide provides instructions on how to deploy the Tour of Beam environment o # Prepare deployment configuration: -``` + 1. Navigate to `beam/learning/tour-of-beam/terraform` ``` @@ -61,8 +61,7 @@ cd beam/learning/tour-of-beam/terraform 2. Configure authentication for the Google Cloud Platform (GCP) ``` gcloud init -``` -``` + gcloud auth application-default login ``` @@ -98,29 +97,157 @@ terraform plan -var gcloud_init_account=$(gcloud config get-value core/account) ``` Where: -- **environment** - environment name +- **environment** - Infrastructure environment name - **region** - GCP region for your infrastructure # Deploy the Tour of Beam Frontend Infrastructure: -8. Run the following command and follow the instructions to configure authentication for Firebase: +8. Update config.dart configuration file under beam/learning/tour-of-beam/frontend/lib: + +Navigate to beam/learning/tour-of-beam/frontend/lib. +Update config.dart file, replacing values in ${ } with your actual values. + +Where: +- **${cloudfunctions_region}** - region where GCP Cloud Functions have been deployed +- **${project_id}** - GCP project where infrastructure being deployed +- **${environment}** - Infrastructure environment name +- **${dns_name}** - DNS record name reserved for Beam Playground environment + +``` +const _cloudFunctionsProjectRegion = '${cloudfunctions_region}'; +const _cloudFunctionsProjectId = '${project_id}'; +const cloudFunctionsBaseUrl = 'https://' + '$_cloudFunctionsProjectRegion-$_cloudFunctionsProjectId' + '.cloudfunctions.net/${environment}_'; + + +const String kAnalyticsUA = 'UA-73650088-2'; +const String kApiClientURL = +'https://router.${dns_name}'; +const String kApiJavaClientURL = +'https://java.${dns_name}'; +const String kApiGoClientURL = +'https://go.${dns_name}'; +const String kApiPythonClientURL = +'https://python.${dns_name}'; +const String kApiScioClientURL = +'https://scio.${dns_name}'; +``` + +9. Create file .firebaserc under beam/learning/tour-of-beam/frontend + +Navigate to beam/learning/tour-of-beam/frontend. +Create .firebaserc file with the following content. + +Where: +- **${project_id}** - GCP project where infrastructure being deployed + +``` +{ +"projects": { +"default": "${project_id}" + } +} +``` + +10. Login into the Firebase CLI + +``` +# To use nteractive mode (forwards to browser webpage from the terminal) +firebase login +``` + ``` +# To use non-interactive mode (generates link) firebase login --no-localhost ``` -7. Run the following command from the top-level repository folder ("beam") to deploy the Tour of Beam Frontend infrastructure: + +11. Create Firebase Project + ``` -./gradlew learning:tour-of-beam:terraform:InitFrontend -Pproject_environment="environment_name" -Pproject_id="gcp-project-id" -Pdns-name="playground-dns-name" -Pregion="gcp-region" -Pwebapp_id="firebase_webapp_name" +firebase projects:addfirebase ``` -Where: -- **project_environment** - environment name -- **project_id** - name of your GCP Project ID -- **dns-name** - DNS name reserved for Beam Playground -- **region** - name of your GCP Resources region -- **webapp_id** - name of your Firebase Web Application that will be created (example: Tour-of-Beam-Web-App) + +12. Create Firebase Web App + +``` +firebase apps:create WEB ${webapp_name} --project ${project_id} +``` + +13. Get Firebase Web App configuration + +Once Firebase Web App has been created, following output will be shown: + +``` +Create your WEB app in project cloudbuild-383310: +✔ Creating your Web app + +🎉🎉🎉 Your Firebase WEB App is ready! 🎉🎉🎉 + +App information: +- App ID: WEBAPP_ID +- Display name: WEBAPP_NAME + +You can run this command to print out your new app's Google Services config: +firebase apps:sdkconfig WEB WEBAPP_ID +``` + +14. Prepare Firebase options configuration file + +Copy and paste into the terminal last line to get Web App configuration. + +Output: + +``` +✔ Downloading configuration data of your Firebase WEB app +// Copy and paste this into your JavaScript code to initialize the Firebase SDK. +// You will also need to load the Firebase SDK. +// See https://firebase.google.com/docs/web/setup for more details. + +firebase.initializeApp({ + "projectId": + "appId": + "storageBucket": + "apiKey": + "authDomain": + "messagingSenderId": +}); +``` + +Copy the lines inside the curly braces. +Paste and replace the data inside the parentheses in beam/learning/tour-of-beam/frontend/lib/firebase_options.dart file. + +``` +static const FirebaseOptions web = FirebaseOptions( + + +); +``` + +15. Run flutter and firebase commands to deploy Tour of Beam frontend + +Navigate to beam/playground/frontend/playground_components and run flutter commands + +``` +# Go to beam/playground/frontend/playground_components first +flutter pub get +flutter pub run build_runner build --delete-conflicting-outputs +``` + +Navigate to beam/learning/tour-of-beam/frontend and run flutter commands + +``` +Go to beam/learning/tour-of-beam/frontend first +flutter pub get +flutter pub run build_runner build --delete-conflicting-outputs +flutter build web --profile --dart-define=Dart2jsOptimization=O0 +firebase deploy --project ${project_id} +``` + # Validate the deployment of the Tour of Beam: -8. Open the Tour of Beam webpage in a web browser (Hosting URL will be provided in terminal output) to ensure that deployment has been successfully completed. +16. Open the Tour of Beam webpage in a web browser (Hosting URL will be provided in terminal output) to ensure that deployment has been successfully completed. Example: ``` From 71c7dfb1f22f4e670bad89c46e5b13d5cd9060a5 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 20 Apr 2023 00:35:36 +0500 Subject: [PATCH 19/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index a862b08a93b8..721b8852dd14 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -105,6 +105,7 @@ Where: 8. Update config.dart configuration file under beam/learning/tour-of-beam/frontend/lib: Navigate to beam/learning/tour-of-beam/frontend/lib. + Update config.dart file, replacing values in ${ } with your actual values. Where: @@ -169,15 +170,13 @@ firebase login --no-localhost firebase projects:addfirebase ``` -12. Create Firebase Web App +12. Create Firebase Web App and prepare Firebase configuration file ``` firebase apps:create WEB ${webapp_name} --project ${project_id} ``` -13. Get Firebase Web App configuration - -Once Firebase Web App has been created, following output will be shown: +Once Firebase Web App has been created, there will be following output example: ``` Create your WEB app in project cloudbuild-383310: @@ -193,11 +192,9 @@ You can run this command to print out your new app's Google Services config: firebase apps:sdkconfig WEB WEBAPP_ID ``` -14. Prepare Firebase options configuration file - Copy and paste into the terminal last line to get Web App configuration. -Output: +Output example: ``` ✔ Downloading configuration data of your Firebase WEB app @@ -216,7 +213,8 @@ firebase.initializeApp({ ``` Copy the lines inside the curly braces. -Paste and replace the data inside the parentheses in beam/learning/tour-of-beam/frontend/lib/firebase_options.dart file. + +Paste (replace) the copied data inside the parentheses in beam/learning/tour-of-beam/frontend/lib/firebase_options.dart file. ``` static const FirebaseOptions web = FirebaseOptions( @@ -225,7 +223,7 @@ static const FirebaseOptions web = FirebaseOptions( ); ``` -15. Run flutter and firebase commands to deploy Tour of Beam frontend +13. Run flutter and firebase commands to deploy Tour of Beam frontend Navigate to beam/playground/frontend/playground_components and run flutter commands @@ -245,9 +243,8 @@ flutter build web --profile --dart-define=Dart2jsOptimization=O0 firebase deploy --project ${project_id} ``` - # Validate the deployment of the Tour of Beam: -16. Open the Tour of Beam webpage in a web browser (Hosting URL will be provided in terminal output) to ensure that deployment has been successfully completed. +14. Open the Tour of Beam webpage in a web browser (Hosting URL will be provided in terminal output) to ensure that deployment has been successfully completed. Example: ``` From 2986675930875df9b452b0ade1017d3cde2db2ed Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 20 Apr 2023 00:37:45 +0500 Subject: [PATCH 20/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 721b8852dd14..241c3e77028a 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -104,9 +104,9 @@ Where: 8. Update config.dart configuration file under beam/learning/tour-of-beam/frontend/lib: -Navigate to beam/learning/tour-of-beam/frontend/lib. + 8.1. Navigate to beam/learning/tour-of-beam/frontend/lib. -Update config.dart file, replacing values in ${ } with your actual values. + 8.2. Update config.dart file, replacing values in ${ } with your actual values. Where: - **${cloudfunctions_region}** - region where GCP Cloud Functions have been deployed @@ -137,8 +137,9 @@ const String kApiScioClientURL = 9. Create file .firebaserc under beam/learning/tour-of-beam/frontend -Navigate to beam/learning/tour-of-beam/frontend. -Create .firebaserc file with the following content. + 9.1. Navigate to beam/learning/tour-of-beam/frontend. + + 9.2. Create .firebaserc file with the following content. Where: - **${project_id}** - GCP project where infrastructure being deployed @@ -244,6 +245,7 @@ firebase deploy --project ${project_id} ``` # Validate the deployment of the Tour of Beam: + 14. Open the Tour of Beam webpage in a web browser (Hosting URL will be provided in terminal output) to ensure that deployment has been successfully completed. Example: From 1f18ff82b4bfde71c77a575874b251df0105649d Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 20 Apr 2023 00:39:31 +0500 Subject: [PATCH 21/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 241c3e77028a..0413d8dcc3b5 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -237,7 +237,7 @@ flutter pub run build_runner build --delete-conflicting-outputs Navigate to beam/learning/tour-of-beam/frontend and run flutter commands ``` -Go to beam/learning/tour-of-beam/frontend first +# Go to beam/learning/tour-of-beam/frontend first flutter pub get flutter pub run build_runner build --delete-conflicting-outputs flutter build web --profile --dart-define=Dart2jsOptimization=O0 From 7aef545ba71eb1a84b2dc3e015918a5c4a66dfef Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 20 Apr 2023 17:37:41 +0500 Subject: [PATCH 22/35] Bulk update of terraform scripts --- .../tour-of-beam/terraform/api_enable/main.tf | 1 + .../terraform/cloud_functions/main.tf | 6 +++-- .../terraform/cloud_functions/variables.tf | 23 ++++++------------- .../terraform/functions_buckets/locals.tf | 2 ++ .../terraform/functions_buckets/main.tf | 4 ++-- .../terraform/functions_buckets/output.tf | 2 ++ .../terraform/functions_buckets/variables.tf | 1 + learning/tour-of-beam/terraform/main.tf | 4 ++++ learning/tour-of-beam/terraform/setup/iam.tf | 3 +++ .../tour-of-beam/terraform/setup/locals.tf | 2 +- .../tour-of-beam/terraform/setup/output.tf | 6 +---- learning/tour-of-beam/terraform/variables.tf | 9 ++++++++ 12 files changed, 37 insertions(+), 26 deletions(-) diff --git a/learning/tour-of-beam/terraform/api_enable/main.tf b/learning/tour-of-beam/terraform/api_enable/main.tf index 9d792c246371..263a6b93b627 100644 --- a/learning/tour-of-beam/terraform/api_enable/main.tf +++ b/learning/tour-of-beam/terraform/api_enable/main.tf @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +# GCP API Services to be enabled resource "google_project_service" "required_services" { for_each = toset([ "cloudresourcemanager", diff --git a/learning/tour-of-beam/terraform/cloud_functions/main.tf b/learning/tour-of-beam/terraform/cloud_functions/main.tf index 1afac61f431e..4a0deaa071ed 100644 --- a/learning/tour-of-beam/terraform/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform/cloud_functions/main.tf @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. - +# GCP Cloud Functions that will serve as a part of the backend of Tour of Beam infrastructure resource "google_cloudfunctions_function" "cloud_function" { count = length(var.entry_point_names) name = "${var.environment}_${var.entry_point_names[count.index]}" @@ -45,7 +45,9 @@ resource "google_cloudfunctions_function" "cloud_function" { } -# Create IAM entry so all users can invoke the function +# Create IAM entry so all users can invoke the cloud functions +# Endpoints serve content only +# Has additional firebase authentication called "Bearer token" for endpoints that update or delete user progress resource "google_cloudfunctions_function_iam_member" "invoker" { count = length(google_cloudfunctions_function.cloud_function) project = var.project_id diff --git a/learning/tour-of-beam/terraform/cloud_functions/variables.tf b/learning/tour-of-beam/terraform/cloud_functions/variables.tf index 0139a5c2a85c..478cdba40e13 100644 --- a/learning/tour-of-beam/terraform/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform/cloud_functions/variables.tf @@ -15,20 +15,6 @@ # specific language governing permissions and limitations # under the License. -# This will be used to generate certain variables values -resource "random_string" "id" { - length = 4 - upper = false - special = false -} - -# This will be used to generate certain variables -variable "resource_name_prefix" { - type = string - description = "The resource name prefix applied to all resource naming for the application" - default = "tour-of-beam" -} - # Taken from output of SETUP module variable "service_account_id" { description = "The name of Service Account to run Cloud Function" @@ -40,15 +26,18 @@ variable "project_id" { description = "The GCP Project ID of function" } +# GCP region variable "region" { description = "The GCP Region where cloud functions will be created" } +# Source code bucket name, used for cloud functions # Taken from output of FUNCTIONS_BUCKETS module variable "source_archive_bucket" { description = "The GCS bucket containing the zip archive which contains the function" } +# Source code objects name, used for cloud functions # Taken from output of FUNCTIONS_BUCKETS module variable "source_archive_object" { description = "The source archive object (file) in archive bucket" @@ -60,12 +49,14 @@ variable "entry_point_names" { default = ["getSdkList", "getContentTree", "getUnitContent", "getUserProgress", "postUnitComplete", "postUserCode", "postDeleteProgress"] } -# Generated during execution Kotlin Gradle script +# Existing Playground environment's router hostname:port details +# Variable assigned using "kubectl" command variable "pg_router_host" { description = "Hostname:port of Playground GKE cluster's router grpc workload" } -# To be provided manually as argument for Kotlin Gradle script +# To support multi environment architecture +# Env name (e.g. test, prod, dev) variable "environment" { description = "The name of the environment for deployment of cloud functions. Will be appended to the name of cloud functions" } diff --git a/learning/tour-of-beam/terraform/functions_buckets/locals.tf b/learning/tour-of-beam/terraform/functions_buckets/locals.tf index e13282ce4c1f..ec2f63cb222e 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/locals.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/locals.tf @@ -15,12 +15,14 @@ # specific language governing permissions and limitations # under the License. +# Random string generator resource. To generate source code buckets name resource "random_string" "id" { length = 4 upper = false special = false } +# Variable for prefix. Used in generated source code buckets name variable "resource_name_prefix" { type = string description = "The resource name prefix applied to all resource naming for the application" diff --git a/learning/tour-of-beam/terraform/functions_buckets/main.tf b/learning/tour-of-beam/terraform/functions_buckets/main.tf index d22dd5dd4a89..6b0992833b92 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/main.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/main.tf @@ -15,14 +15,14 @@ # specific language governing permissions and limitations # under the License. -# This creates GCS bucket for source code for cloud functions +# GCS bucket for source code for cloud functions resource "google_storage_bucket" "cloud_functions_bucket" { name = local.cloudfunctions_bucket location = var.region storage_class = "STANDARD" } -# This creates object for a bucket with the source code +# GCS bucket object to store source code resource "google_storage_bucket_object" "zip" { # Use an MD5 here. If there's no changes to the source code, this won't change either. # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing diff --git a/learning/tour-of-beam/terraform/functions_buckets/output.tf b/learning/tour-of-beam/terraform/functions_buckets/output.tf index 653466fca27c..ee858700dce3 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/output.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/output.tf @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +# Outputs to be used in cloud_function module output "functions-bucket-name" { value = google_storage_bucket.cloud_functions_bucket.name } @@ -23,6 +24,7 @@ output "function-bucket-object" { value = google_storage_bucket_object.zip.name } +# Output to be used as variable for google_storage_bucket resource output "cloudfunctions-bucket-name" { value = local.cloudfunctions_bucket } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf b/learning/tour-of-beam/terraform/functions_buckets/variables.tf index c71ea1f36615..f1cb931f2a2b 100644 --- a/learning/tour-of-beam/terraform/functions_buckets/variables.tf +++ b/learning/tour-of-beam/terraform/functions_buckets/variables.tf @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +# GCP region where GCS bucket will be created variable "region" { description = "The GCP region where GCS bucket will be created (For Cloud Functions source code)" } diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf index 75a3896835e3..c159053e2eb7 100644 --- a/learning/tour-of-beam/terraform/main.tf +++ b/learning/tour-of-beam/terraform/main.tf @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +# Setup module to create service accounts, assign required IAM roles to it and deploying user account module "setup" { source = "./setup" project_id = var.project_id @@ -22,16 +23,19 @@ module "setup" { depends_on = [module.api_enable] } +# GCS buckets to create buckets, objects, archive to store source code that cloud functions will use module "functions_buckets" { source = "./functions_buckets" region = var.region depends_on = [module.setup, module.api_enable] } +# API services module. Enables required APIs for the infrastructure module "api_enable" { source = "./api_enable" } +# Cloud functions module. Creates cloud functions, as part of Tour of Beam backend infrastructure module "cloud_functions" { source = "./cloud_functions" region = var.region diff --git a/learning/tour-of-beam/terraform/setup/iam.tf b/learning/tour-of-beam/terraform/setup/iam.tf index 08ac5e8bacba..245872364fd6 100644 --- a/learning/tour-of-beam/terraform/setup/iam.tf +++ b/learning/tour-of-beam/terraform/setup/iam.tf @@ -15,11 +15,13 @@ # specific language governing permissions and limitations # under the License. +# Service account for GCP Cloud Functions resource "google_service_account" "cloud_function_sa" { account_id = local.cloudfunctions_service_account display_name = "Service Account to run Cloud Functions" } +# IAM roles for Cloud Functions service account resource "google_project_iam_member" "terraform_service_account_roles" { for_each = toset([ "roles/cloudfunctions.admin", "roles/storage.objectViewer", @@ -31,6 +33,7 @@ resource "google_project_iam_member" "terraform_service_account_roles" { project = var.project_id } +# IAM roles to be granted for user account that will be running terraform scripts resource "google_project_iam_member" "gcloud_user_required_roles" { for_each = toset([ "roles/cloudfunctions.admin", "roles/firebase.admin" diff --git a/learning/tour-of-beam/terraform/setup/locals.tf b/learning/tour-of-beam/terraform/setup/locals.tf index 5c0681c9fb14..9817b8f2821d 100644 --- a/learning/tour-of-beam/terraform/setup/locals.tf +++ b/learning/tour-of-beam/terraform/setup/locals.tf @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -# Local value to store generated Cloud Functions Service account name +# Local value to store generated Cloud Functions' Service account name resource "random_string" "id" { length = 4 diff --git a/learning/tour-of-beam/terraform/setup/output.tf b/learning/tour-of-beam/terraform/setup/output.tf index 3adca0c186c8..bd069f1ed8d8 100644 --- a/learning/tour-of-beam/terraform/setup/output.tf +++ b/learning/tour-of-beam/terraform/setup/output.tf @@ -15,11 +15,7 @@ # specific language governing permissions and limitations # under the License. -# This output required to assign service account to cloud functions +# Output used to assign service account to cloud functions output "service-account-email" { value = google_service_account.cloud_function_sa.email -} - -output "cloudfunctions-service-account" { - value = local.cloudfunctions_service_account } \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf index d9debeeba851..0e34fbb26f2b 100644 --- a/learning/tour-of-beam/terraform/variables.tf +++ b/learning/tour-of-beam/terraform/variables.tf @@ -15,22 +15,31 @@ # specific language governing permissions and limitations # under the License. +# GCP Project ID +# Provided as a result of gcloud command variable "project_id" { description = "The ID of the Google Cloud project within which resources are provisioned" } +# GCP Region where infrastructure will be deployed variable "region" { description = "The region of the Google Cloud project within which resources are provisioned" } +# User account that will be deploying Tour of Beam infrastructure +# Provided as a result of gcloud command variable "gcloud_init_account" { description = "User Account ID logged in with gcloud init command (e.g. username@domain.com)" } +# Existing Playground router hostname:port details +# Provided as a result of kubectl command variable "pg_router_host" { description = "Hostname:port of Playground GKE cluster's router grpc workload" } +# Variable for multi-environment +# To create env (e.g. prod, dev, test) variable "environment" { description = "The name of the environment for deployment. Will create directory where terraform config files will be stored" } From 73d8825ca6c788767c853515b728a60c430e1612 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 21 Apr 2023 17:43:42 +0500 Subject: [PATCH 23/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 0413d8dcc3b5..82ad55a1cd75 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -36,14 +36,12 @@ This guide provides instructions on how to deploy the Tour of Beam environment o 4. An OS with the following software installed: -* [Java](https://adoptopenjdk.net/) * [Flutter (3.7.3 >)](https://docs.flutter.dev/get-started/install) * [Dart SDK (2.19.2)](https://dart.dev/get-dart) * [Firebase-tools CLI](https://www.npmjs.com/package/firebase-tools) * [Terraform](https://www.terraform.io/downloads) * [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) * [Kubectl authentication plugin](https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke) -* [Go](https://go.dev/doc/install) 5. Beam Playground environment (Existing GKE Cluster will be required particularly) From 9c2cd04d3b73036d952d125fe9622282d805c43e Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 26 Apr 2023 01:43:52 +0500 Subject: [PATCH 24/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 82ad55a1cd75..f3c257410069 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -17,7 +17,8 @@ under the License. --> # The Tour of Beam deployment on GCP -This guide provides instructions on how to deploy the Tour of Beam environment on Google Cloud Platform (GCP) and Firebase environment. Before starting the deployment, ensure that you have the following prerequisites in place: +This guide provides instructions on how to deploy the Tour of Beam environment on Google Cloud Platform (GCP) and Firebase environment. +Before starting the deployment, ensure that you have the following prerequisites in place: ## Prerequisites: @@ -32,7 +33,7 @@ This guide provides instructions on how to deploy the Tour of Beam environment o - Storage Admin - Kubernetes Engine Cluster Viewer -3. [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for saving deployment state +3. [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for saving deployment state 4. An OS with the following software installed: @@ -43,20 +44,21 @@ This guide provides instructions on how to deploy the Tour of Beam environment o * [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) * [Kubectl authentication plugin](https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke) -5. Beam Playground environment (Existing GKE Cluster will be required particularly) +5. Existing Beam Playground environment 6. Apache Beam Git repository cloned locally # Prepare deployment configuration: -1. Navigate to `beam/learning/tour-of-beam/terraform` +1. Navigate to Apache Beam cloned repository's `beam/learning/tour-of-beam/terraform` directory ``` cd beam/learning/tour-of-beam/terraform ``` -2. Configure authentication for the Google Cloud Platform (GCP) +2. Configure authentication for the Google Cloud Platform (GCP). _(Note: Authentication to the GCP Project required to run gcloud commands)_
+ ``` gcloud init @@ -68,7 +70,7 @@ gcloud auth application-default login gcloud auth configure-docker `chosen_region`-docker.pkg.dev ``` -4. And the authentication in GCP Google Kubernetes Engine: +4. And the authentication in GCP Google Kubernetes Engine: _(Note: Authentication to docker and GKE required to fetch GRPC router ip:port info)_
``` gcloud container clusters get-credentials --region `chosen_gke_zone` `gke_name` --project `project_id` ``` From 633b4adabed6d75ff5f1131f6f0e64eb2340acfd Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 26 Apr 2023 02:13:07 +0500 Subject: [PATCH 25/35] Datastore_namespace updates --- learning/tour-of-beam/terraform/README.md | 11 +++++++++++ .../tour-of-beam/terraform/cloud_functions/main.tf | 1 + .../terraform/cloud_functions/variables.tf | 4 ++++ learning/tour-of-beam/terraform/variables.tf | 4 ++++ 4 files changed, 20 insertions(+) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index f3c257410069..2b38970d277a 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -88,11 +88,22 @@ terraform init -backend-config="bucket=`created_gcs_bucket`" ``` 7. Run terraform apply to create Tour-Of-Beam backend infrastructure + +``` +terraform plan -var gcloud_init_account=$(gcloud config get-value core/account) \ +-var environment="test" \ +-var region="us-east1" \ +-var project_id=$(gcloud config get-value project) \ +-var datastore_namespace = "namespace" \ +-var pg_router_host=$(kubectl get svc -l app=backend-router-grpc -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}') +``` + ``` terraform plan -var gcloud_init_account=$(gcloud config get-value core/account) \ -var environment="test" \ -var region="us-east1" \ -var project_id=$(gcloud config get-value project) \ +-var datastore_namespace = "namespace" \ -var pg_router_host=$(kubectl get svc -l app=backend-router-grpc -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}') ``` diff --git a/learning/tour-of-beam/terraform/cloud_functions/main.tf b/learning/tour-of-beam/terraform/cloud_functions/main.tf index 4a0deaa071ed..e85d67cd3c6d 100644 --- a/learning/tour-of-beam/terraform/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform/cloud_functions/main.tf @@ -36,6 +36,7 @@ resource "google_cloudfunctions_function" "cloud_function" { DATASTORE_PROJECT_ID=var.project_id GOOGLE_PROJECT_ID=var.project_id PLAYGROUND_ROUTER_HOST=var.pg_router_host + DATASTORE_NAMESPACE=var.datastore_namespace } timeouts { diff --git a/learning/tour-of-beam/terraform/cloud_functions/variables.tf b/learning/tour-of-beam/terraform/cloud_functions/variables.tf index 478cdba40e13..c7442359de62 100644 --- a/learning/tour-of-beam/terraform/cloud_functions/variables.tf +++ b/learning/tour-of-beam/terraform/cloud_functions/variables.tf @@ -60,3 +60,7 @@ variable "pg_router_host" { variable "environment" { description = "The name of the environment for deployment of cloud functions. Will be appended to the name of cloud functions" } + +variable "datastore_namespace" { + description = "The name of datastore namespace" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf index 0e34fbb26f2b..0a045514dd5b 100644 --- a/learning/tour-of-beam/terraform/variables.tf +++ b/learning/tour-of-beam/terraform/variables.tf @@ -43,3 +43,7 @@ variable "pg_router_host" { variable "environment" { description = "The name of the environment for deployment. Will create directory where terraform config files will be stored" } + +variable "datastore_namespace" { + description = "The name of datastore namespace" +} \ No newline at end of file From f1193f2aa9187e9fe461a1174c9b6e3ea5227c01 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 26 Apr 2023 20:28:09 +0500 Subject: [PATCH 26/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 2b38970d277a..7bbfc01be0a5 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -17,7 +17,7 @@ under the License. --> # The Tour of Beam deployment on GCP -This guide provides instructions on how to deploy the Tour of Beam environment on Google Cloud Platform (GCP) and Firebase environment. +This guide provides instructions on how to deploy the Tour of Beam environment on Google Cloud Platform (GCP) and Firebase environment. Before starting the deployment, ensure that you have the following prerequisites in place: ## Prerequisites: @@ -33,7 +33,7 @@ Before starting the deployment, ensure that you have the following prerequisites - Storage Admin - Kubernetes Engine Cluster Viewer -3. [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for saving deployment state +3. [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for saving deployment state 4. An OS with the following software installed: @@ -99,7 +99,7 @@ terraform plan -var gcloud_init_account=$(gcloud config get-value core/account) ``` ``` -terraform plan -var gcloud_init_account=$(gcloud config get-value core/account) \ +terraform apply -var gcloud_init_account=$(gcloud config get-value core/account) \ -var environment="test" \ -var region="us-east1" \ -var project_id=$(gcloud config get-value project) \ @@ -167,7 +167,7 @@ Where: ``` # To use nteractive mode (forwards to browser webpage from the terminal) -firebase login +firebase login ``` ``` @@ -215,12 +215,12 @@ Output example: // See https://firebase.google.com/docs/web/setup for more details. firebase.initializeApp({ - "projectId": - "appId": - "storageBucket": - "apiKey": - "authDomain": - "messagingSenderId": + "projectId": + "appId": + "storageBucket": + "apiKey": + "authDomain": + "messagingSenderId": }); ``` From 3bdd595e9001c1bdcf3035a722ad31635706fcfa Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Wed, 26 Apr 2023 21:06:59 +0500 Subject: [PATCH 27/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 7bbfc01be0a5..6a10a5baf133 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -148,7 +148,7 @@ const String kApiScioClientURL = 9. Create file .firebaserc under beam/learning/tour-of-beam/frontend - 9.1. Navigate to beam/learning/tour-of-beam/frontend. + 9.1. Navigate to beam/learning/tour-of-beam/frontend. 9.2. Create .firebaserc file with the following content. @@ -235,7 +235,7 @@ static const FirebaseOptions web = FirebaseOptions( ); ``` -13. Run flutter and firebase commands to deploy Tour of Beam frontend +13. Run flutter and firebase commands to deploy Tour of Beam frontend Navigate to beam/playground/frontend/playground_components and run flutter commands From 8146919f36a4d0d3e89c3ccc4d28240b3a182538 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 27 Apr 2023 00:40:31 +0500 Subject: [PATCH 28/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 6a10a5baf133..b35a64378653 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -241,7 +241,7 @@ Navigate to beam/playground/frontend/playground_components and run flutter comma ``` # Go to beam/playground/frontend/playground_components first -flutter pub get +flutter pub get flutter pub run build_runner build --delete-conflicting-outputs ``` @@ -249,7 +249,7 @@ Navigate to beam/learning/tour-of-beam/frontend and run flutter commands ``` # Go to beam/learning/tour-of-beam/frontend first -flutter pub get +flutter pub get flutter pub run build_runner build --delete-conflicting-outputs flutter build web --profile --dart-define=Dart2jsOptimization=O0 firebase deploy --project ${project_id} From 1e680536d836486b0f51b14f9da57e5905091487 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 27 Apr 2023 14:17:42 +0500 Subject: [PATCH 29/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index b35a64378653..c9d226d73c9d 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -110,6 +110,7 @@ terraform apply -var gcloud_init_account=$(gcloud config get-value core/account) Where: - **environment** - Infrastructure environment name - **region** - GCP region for your infrastructure +- **datastore_namespace** - Beam Playground Datastore's namespace # Deploy the Tour of Beam Frontend Infrastructure: From 87e02d8f78132bcf705b4879a62de504a9c9cad1 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 27 Apr 2023 22:37:43 +0500 Subject: [PATCH 30/35] Update main.tf --- learning/tour-of-beam/terraform/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf index c159053e2eb7..ba122fc7acb8 100644 --- a/learning/tour-of-beam/terraform/main.tf +++ b/learning/tour-of-beam/terraform/main.tf @@ -42,6 +42,7 @@ module "cloud_functions" { project_id = var.project_id pg_router_host = var.pg_router_host environment = var.environment + datastore_namespace = var.datastore_namespace service_account_id = module.setup.service-account-email source_archive_bucket = module.functions_buckets.functions-bucket-name source_archive_object = module.functions_buckets.function-bucket-object From dff9e0ba62a863bda03a563cf4e19cf70fe6d48a Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 27 Apr 2023 23:42:14 +0500 Subject: [PATCH 31/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 51 +++++++++++++---------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index c9d226d73c9d..491a0e5c0f75 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -90,21 +90,21 @@ terraform init -backend-config="bucket=`created_gcs_bucket`" 7. Run terraform apply to create Tour-Of-Beam backend infrastructure ``` -terraform plan -var gcloud_init_account=$(gcloud config get-value core/account) \ --var environment="test" \ --var region="us-east1" \ --var project_id=$(gcloud config get-value project) \ --var datastore_namespace = "namespace" \ --var pg_router_host=$(kubectl get svc -l app=backend-router-grpc -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}') +terraform plan -var "gcloud_init_account=$(gcloud config get-value core/account)" \ +-var "environment=prod" \ +-var "region=us-west1" \ +-var "project_id=$(gcloud config get-value project)" \ +-var "datastore_namespace=playground-datastore-namespace" \ +-var "pg_router_host=$(kubectl get svc -l app=backend-router-grpc -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}')" ``` ``` -terraform apply -var gcloud_init_account=$(gcloud config get-value core/account) \ --var environment="test" \ --var region="us-east1" \ --var project_id=$(gcloud config get-value project) \ --var datastore_namespace = "namespace" \ --var pg_router_host=$(kubectl get svc -l app=backend-router-grpc -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}') +terraform apply -var "gcloud_init_account=$(gcloud config get-value core/account)" \ +-var "environment=prod" \ +-var "region=us-west1" \ +-var "project_id=$(gcloud config get-value project)" \ +-var "datastore_namespace=playground-datastore-namespace" \ +-var "pg_router_host=$(kubectl get svc -l app=backend-router-grpc -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}')" ``` Where: @@ -167,7 +167,7 @@ Where: 10. Login into the Firebase CLI ``` -# To use nteractive mode (forwards to browser webpage from the terminal) +# To use an interactive mode (forwards to a browser webpage) firebase login ``` @@ -186,7 +186,7 @@ firebase projects:addfirebase 12. Create Firebase Web App and prepare Firebase configuration file ``` -firebase apps:create WEB ${webapp_name} --project ${project_id} +firebase apps:create WEB ${webapp_name} --project=$(gcloud config get-value project) ``` Once Firebase Web App has been created, there will be following output example: @@ -216,18 +216,25 @@ Output example: // See https://firebase.google.com/docs/web/setup for more details. firebase.initializeApp({ - "projectId": - "appId": - "storageBucket": - "apiKey": - "authDomain": - "messagingSenderId": + "projectId": "cloudbuild-384304", + "appId": "1:1111111111:web:111111111111", + "storageBucket": "cloudbuild-384304.appspot.com", + "locationId": "us-west1", + "apiKey": "someApiKey", + "authDomain": "cloudbuild-384304.firebaseapp.com", + "messagingSenderId": "111111111111" }); ``` -Copy the lines inside the curly braces. +Copy the lines inside the curly braces and redact them. -Paste (replace) the copied data inside the parentheses in beam/learning/tour-of-beam/frontend/lib/firebase_options.dart file. +You will need to: + +1) Remove "locationId" line. +2) Remove quotes (") from key of "key": "value" pair. + 3) E.g. `projectId: "cloudbuild-384304"` + +Paste (replace) the redacted data inside the parentheses in beam/learning/tour-of-beam/frontend/lib/firebase_options.dart file. ``` static const FirebaseOptions web = FirebaseOptions( From d1b901160f0c1f4683f92e7a44ebf72140344b09 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 27 Apr 2023 23:45:38 +0500 Subject: [PATCH 32/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 491a0e5c0f75..885d00edb714 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -260,7 +260,7 @@ Navigate to beam/learning/tour-of-beam/frontend and run flutter commands flutter pub get flutter pub run build_runner build --delete-conflicting-outputs flutter build web --profile --dart-define=Dart2jsOptimization=O0 -firebase deploy --project ${project_id} +firebase deploy --project=$(gcloud config get-value project) ``` # Validate the deployment of the Tour of Beam: From 6d91188a8f15c0b21bd00077f1e7f489c7419f21 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Thu, 27 Apr 2023 23:48:21 +0500 Subject: [PATCH 33/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 885d00edb714..29e6499aec84 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -233,6 +233,16 @@ You will need to: 1) Remove "locationId" line. 2) Remove quotes (") from key of "key": "value" pair. 3) E.g. `projectId: "cloudbuild-384304"` +4) In overall, redacted and ready to be inserted data should be as follows: + +``` + projectId: "cloudbuild-384304", + appId: "1:1111111111:web:111111111111", + storageBucket: "cloudbuild-384304.appspot.com", + apiKey: "someApiKey", + authDomain: "cloudbuild-384304.firebaseapp.com", + messagingSenderId: "111111111111" +``` Paste (replace) the redacted data inside the parentheses in beam/learning/tour-of-beam/frontend/lib/firebase_options.dart file. From d4c09c26402b56bdc570b70cafc61dff412e82b5 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 28 Apr 2023 16:17:03 +0500 Subject: [PATCH 34/35] Some minor TF updates --- learning/tour-of-beam/terraform/cloud_functions/main.tf | 1 + learning/tour-of-beam/terraform/provider.tf | 2 ++ 2 files changed, 3 insertions(+) diff --git a/learning/tour-of-beam/terraform/cloud_functions/main.tf b/learning/tour-of-beam/terraform/cloud_functions/main.tf index e85d67cd3c6d..e80225115dbe 100644 --- a/learning/tour-of-beam/terraform/cloud_functions/main.tf +++ b/learning/tour-of-beam/terraform/cloud_functions/main.tf @@ -47,6 +47,7 @@ resource "google_cloudfunctions_function" "cloud_function" { } # Create IAM entry so all users can invoke the cloud functions + # Endpoints serve content only # Has additional firebase authentication called "Bearer token" for endpoints that update or delete user progress resource "google_cloudfunctions_function_iam_member" "invoker" { diff --git a/learning/tour-of-beam/terraform/provider.tf b/learning/tour-of-beam/terraform/provider.tf index df421ee1fad4..127c7d14a4a7 100644 --- a/learning/tour-of-beam/terraform/provider.tf +++ b/learning/tour-of-beam/terraform/provider.tf @@ -17,6 +17,7 @@ # under the License. # +# Terraform state to be saved in GCS bucket terraform { backend "gcs" { } @@ -29,6 +30,7 @@ terraform { } } +# GCP Provider resource provider "google" { project = var.project_id region = var.region From 284148a73af1047896f9fb4ac362ecae112c9ed8 Mon Sep 17 00:00:00 2001 From: ruslan-ikhsan Date: Fri, 28 Apr 2023 16:24:59 +0500 Subject: [PATCH 35/35] Update README.md --- learning/tour-of-beam/terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md index 29e6499aec84..24b4d83e67ea 100644 --- a/learning/tour-of-beam/terraform/README.md +++ b/learning/tour-of-beam/terraform/README.md @@ -231,7 +231,7 @@ Copy the lines inside the curly braces and redact them. You will need to: 1) Remove "locationId" line. -2) Remove quotes (") from key of "key": "value" pair. +2) Remove quotes (") from key of "key": "value" pair. 3) E.g. `projectId: "cloudbuild-384304"` 4) In overall, redacted and ready to be inserted data should be as follows: