From e590fa0124087636794cd751f1b2b4a70fb02bde Mon Sep 17 00:00:00 2001 From: Timur Sultanov Date: Thu, 4 May 2023 17:26:29 +0400 Subject: [PATCH 1/3] Fix initialization-time execution of tasks which should only be executed fruing build time --- .../tour-of-beam/terraform/build.gradle.kts | 343 ++++++++++-------- 1 file changed, 192 insertions(+), 151 deletions(-) diff --git a/learning/tour-of-beam/terraform/build.gradle.kts b/learning/tour-of-beam/terraform/build.gradle.kts index ed4ccc7b1094..054ce826ec29 100644 --- a/learning/tour-of-beam/terraform/build.gradle.kts +++ b/learning/tour-of-beam/terraform/build.gradle.kts @@ -32,214 +32,251 @@ terraformPlugin { tasks.register("terraformInit") { // exec args can be passed by commandline, for example args( - "init", "-migrate-state", - "-backend-config=./state.tfbackend" + "init", "-migrate-state", + "-backend-config=./state.tfbackend" ) } - /* refresh Infrastucture for remote state */ +/* refresh Infrastucture for remote state */ tasks.register("terraformRef") { - args( - "refresh", - "-lock=false", - "-var-file=./common.tfvars" - ) - } + args( + "refresh", + "-lock=false", + "-var-file=./common.tfvars" + ) +} tasks.register("terraformApplyBackend") { - group = "backend-deploy" - var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String - args( - "apply", - "-auto-approve", - "-lock=false", - "-parallelism=3", - "-var=pg_router_host=$pg_router_host", - "-var=gcloud_init_account=$(gcloud config get-value core/account)", - "-var=project_id=$(gcloud config get-value project)", - "-var-file=./common.tfvars" - ) + group = "backend-deploy" + var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String + args( + "apply", + "-auto-approve", + "-lock=false", + "-parallelism=3", + "-var=pg_router_host=$pg_router_host", + "-var=gcloud_init_account=$(gcloud config get-value core/account)", + "-var=project_id=$(gcloud config get-value project)", + "-var-file=./common.tfvars" + ) tasks.getByName("uploadLearningMaterials").mustRunAfter(this) - } +} tasks.register("terraformDestroy") { var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String args( - "destroy", - "-auto-approve", - "-lock=false", - "-var=pg_router_host=$pg_router_host", - "-var=gcloud_init_account=$(gcloud config get-value core/account)", - "-var=project_id=$(gcloud config get-value project)", - "-var-file=./common.tfvars" + "destroy", + "-auto-approve", + "-lock=false", + "-var=pg_router_host=$pg_router_host", + "-var=gcloud_init_account=$(gcloud config get-value core/account)", + "-var=project_id=$(gcloud config get-value project)", + "-var-file=./common.tfvars" ) } tasks.register("getRouterHost") { - group = "backend-deploy" - val result = ByteArrayOutputStream() - exec { - commandLine("kubectl", "get", "svc", "-l", "app=backend-router-grpc", "-o", "jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}'") - standardOutput = result + doLast { + group = "backend-deploy" + val result = ByteArrayOutputStream() + exec { + commandLine( + "kubectl", + "get", + "svc", + "-l", + "app=backend-router-grpc", + "-o", + "jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}'" + ) + standardOutput = result + } + val pg_router_host = result.toString().trim().replace("'", "") + project.extensions.extraProperties["pg_router_host"] = pg_router_host } - val pg_router_host = result.toString().trim().replace("'", "") - project.extensions.extraProperties["pg_router_host"] = pg_router_host } tasks.register("indexcreate") { - group = "backend-deploy" - val indexpath = "../backend/internal/storage/index.yaml" - exec { - executable("gcloud") - args("datastore", "indexes", "create", indexpath) + doLast { + group = "backend-deploy" + val indexpath = "../backend/internal/storage/index.yaml" + exec { + executable("gcloud") + args("datastore", "indexes", "create", indexpath) + } } } tasks.register("firebaseProjectCreate") { - group = "frontend-deploy" - val result = ByteArrayOutputStream() - var project_id = project.property("project_id") as String - exec { - executable("firebase") - args("projects:list") - standardOutput = result - } - val output = result.toString().trim() - if (output.contains(project_id)) { - println("Firebase is already added to project $project_id.") - } else { + doLast { + group = "frontend-deploy" + val result = ByteArrayOutputStream() + var project_id = project.property("project_id") as String exec { executable("firebase") - args("projects:addfirebase", project_id) - }.assertNormalExitValue() - println("Firebase has been added to project $project_id.") + args("projects:list") + standardOutput = result + } + val output = result.toString().trim() + if (output.contains(project_id)) { + println("Firebase is already added to project $project_id.") + } else { + exec { + executable("firebase") + args("projects:addfirebase", project_id) + }.assertNormalExitValue() + println("Firebase has been added to project $project_id.") + } } } tasks.register("firebaseWebAppCreate") { - group = "frontend-deploy" - val result = ByteArrayOutputStream() - var project_id = project.property("project_id") as String - var webapp_id = project.property("webapp_id") as String - exec { - executable("firebase") - args("apps:list", "--project", project_id) - standardOutput = result - } - println(result) - val output = result.toString() - if (output.contains(webapp_id)) { - println("Webapp id $webapp_id is already created on the project: $project_id.") - val regex = Regex("$webapp_id[│ ]+([\\w:]+)[│ ]+WEB[│ ]+") - val firebaseAppId = regex.find(output)?.groupValues?.get(1)?.trim() - project.extensions.extraProperties["firebaseAppId"] = firebaseAppId - } else { - val result2 = ByteArrayOutputStream() + doLast { + group = "frontend-deploy" + val result = ByteArrayOutputStream() + var project_id = project.property("project_id") as String + var webapp_id = project.property("webapp_id") as String exec { executable("firebase") - args("apps:create", "WEB", webapp_id, "--project", project_id) - standardOutput = result2 - }.assertNormalExitValue() - val firebaseAppId = result2.toString().lines().find { it.startsWith(" - App ID:") }?.substringAfter(":")?.trim() - project.extensions.extraProperties["firebaseAppId"] = firebaseAppId - println("Firebase app ID for newly created Firebase Web App: $firebaseAppId") + args("apps:list", "--project", project_id) + standardOutput = result + } + println(result) + val output = result.toString() + if (output.contains(webapp_id)) { + println("Webapp id $webapp_id is already created on the project: $project_id.") + val regex = Regex("$webapp_id[│ ]+([\\w:]+)[│ ]+WEB[│ ]+") + val firebaseAppId = regex.find(output)?.groupValues?.get(1)?.trim() + project.extensions.extraProperties["firebaseAppId"] = firebaseAppId + } else { + val result2 = ByteArrayOutputStream() + exec { + executable("firebase") + args("apps:create", "WEB", webapp_id, "--project", project_id) + standardOutput = result2 + }.assertNormalExitValue() + val firebaseAppId = + result2.toString().lines().find { it.startsWith(" - App ID:") }?.substringAfter(":")?.trim() + project.extensions.extraProperties["firebaseAppId"] = firebaseAppId + println("Firebase app ID for newly created Firebase Web App: $firebaseAppId") + } } } // firebase apps:sdkconfig WEB AppId tasks.register("getSdkConfigWebApp") { - group = "frontend-deploy" - val firebaseAppId = project.extensions.extraProperties["firebaseAppId"] as String - val result = ByteArrayOutputStream() - exec { - executable("firebase") - args("apps:sdkconfig", "WEB", firebaseAppId) - standardOutput = result - } - val output = result.toString().trim() - val pattern = Pattern.compile("\\{[^{]*\"locationId\":\\s*\".*?\"[^}]*\\}", Pattern.DOTALL) - val matcher = pattern.matcher(output) - if (matcher.find()) { - val firebaseConfigData = matcher.group().replace("{", "") + doLast { + group = "frontend-deploy" + val firebaseAppId = project.extensions.extraProperties["firebaseAppId"] as String + val result = ByteArrayOutputStream() + exec { + executable("firebase") + args("apps:sdkconfig", "WEB", firebaseAppId) + standardOutput = result + } + val output = result.toString().trim() + val pattern = Pattern.compile("\\{[^{]*\"locationId\":\\s*\".*?\"[^}]*\\}", Pattern.DOTALL) + val matcher = pattern.matcher(output) + if (matcher.find()) { + val firebaseConfigData = matcher.group().replace("{", "") .replace("}", "") .replace("\"locationId\":\\s*\".*?\",?".toRegex(), "") .replace("\"(\\w+)\":".toRegex(), "$1:") .replace(":\\s*\"(.*?)\"".toRegex(), ":\"$1\"") - project.extensions.extraProperties["firebaseConfigData"] = firebaseConfigData.trim() - println("Firebase config data: $firebaseConfigData") + project.extensions.extraProperties["firebaseConfigData"] = firebaseConfigData.trim() + println("Firebase config data: $firebaseConfigData") + } } } tasks.register("prepareFirebaseOptionsDart") { - group = "frontend-deploy" - val firebaseConfigData = project.extensions.extraProperties["firebaseConfigData"] as String - val file = project.file("../frontend/lib/firebase_options.dart") - val content = file.readText() - val updatedContent = content.replace(Regex("""static const FirebaseOptions web = FirebaseOptions\(([^)]+)\);"""), "static const FirebaseOptions web = FirebaseOptions(${firebaseConfigData});") - file.writeText(updatedContent) + doLast { + group = "frontend-deploy" + val firebaseConfigData = project.extensions.extraProperties["firebaseConfigData"] as String + val file = project.file("../frontend/lib/firebase_options.dart") + val content = file.readText() + val updatedContent = content.replace( + Regex("""static const FirebaseOptions web = FirebaseOptions\(([^)]+)\);"""), + "static const FirebaseOptions web = FirebaseOptions(${firebaseConfigData});" + ) + file.writeText(updatedContent) + } } tasks.register("flutterPubGetPG") { - exec { - executable("flutter") - args("pub", "get") - workingDir("../../../playground/frontend/playground_components") + doLast { + exec { + executable("flutter") + args("pub", "get") + workingDir("../../../playground/frontend/playground_components") + } } } tasks.register("flutterPubRunPG") { - exec { - executable("flutter") - args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") - workingDir("../../../playground/frontend/playground_components") + doLast { + exec { + executable("flutter") + args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") + workingDir("../../../playground/frontend/playground_components") + } } } tasks.register("flutterPubGetTob") { - exec { - executable("flutter") - args("pub", "get") - workingDir("../frontend") + doLast { + exec { + executable("flutter") + args("pub", "get") + workingDir("../frontend") + } } } tasks.register("flutterPubRunTob") { - exec { - executable("flutter") - args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") - workingDir("../frontend") + doLast { + exec { + executable("flutter") + args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") + workingDir("../frontend") + } } } tasks.register("flutterBuildWeb") { - exec { - executable("flutter") - args("build", "web", "--profile", "--dart-define=Dart2jsOptimization=O0") - workingDir("../frontend") + doLast { + exec { + executable("flutter") + args("build", "web", "--profile", "--dart-define=Dart2jsOptimization=O0") + workingDir("../frontend") + } } } tasks.register("firebaseDeploy") { - var project_id = project.property("project_id") as String - exec { - commandLine("firebase", "deploy", "--project", project_id) - workingDir("../frontend") + doLast { + var project_id = project.property("project_id") as String + exec { + commandLine("firebase", "deploy", "--project", project_id) + workingDir("../frontend") + } } } tasks.register("prepareConfig") { - group = "frontend-deploy" - var region = project.property("region") as String - var project_id = project.property("project_id") as String - var environment = project.property("project_environment") as String - var dns_name = project.property("dns-name") as String - val configFileName = "config.dart" - val modulePath = project(":learning:tour-of-beam:frontend").projectDir.absolutePath - val file = File("$modulePath/lib/$configFileName") - - file.writeText( + doLast { + group = "frontend-deploy" + var region = project.property("region") as String + var project_id = project.property("project_id") as String + var environment = project.property("project_environment") as String + var dns_name = project.property("dns-name") as String + val configFileName = "config.dart" + val modulePath = project(":learning:tour-of-beam:frontend").projectDir.absolutePath + val file = File("$modulePath/lib/$configFileName") + + file.writeText( """ const _cloudFunctionsProjectRegion = '$region'; const _cloudFunctionsProjectId = '$project_id'; @@ -261,16 +298,18 @@ const String kApiScioClientURL = 'https://scio.${dns_name}'; """ ) + } } tasks.register("prepareFirebasercConfig") { - group = "frontend-deploy" - var project_id = project.property("project_id") as String - val configFileName = ".firebaserc" - val modulePath = project(":learning:tour-of-beam:frontend").projectDir.absolutePath - val file = File("$modulePath/$configFileName") - - file.writeText( + doLast { + group = "frontend-deploy" + var project_id = project.property("project_id") as String + val configFileName = ".firebaserc" + val modulePath = project(":learning:tour-of-beam:frontend").projectDir.absolutePath + val file = File("$modulePath/$configFileName") + + file.writeText( """ { "projects": { @@ -279,17 +318,20 @@ tasks.register("prepareFirebasercConfig") { } """ ) + } } tasks.register("uploadLearningMaterials") { - var project_id = project.property("project_id") as String - group = "backend-deploy" - exec { - commandLine("go", "run", "cmd/ci_cd/ci_cd.go") - environment("DATASTORE_PROJECT_ID", project_id) - environment("GOOGLE_PROJECT_ID", project_id) - environment("TOB_LEARNING_ROOT", "../learning-content/") - workingDir("../backend") + doLast { + var project_id = project.property("project_id") as String + group = "backend-deploy" + exec { + commandLine("go", "run", "cmd/ci_cd/ci_cd.go") + environment("DATASTORE_PROJECT_ID", project_id) + environment("GOOGLE_PROJECT_ID", project_id) + environment("TOB_LEARNING_ROOT", "../learning-content/") + workingDir("../backend") + } } dependsOn("terraformApplyBackend") mustRunAfter("terraformApplyBackend") @@ -312,7 +354,6 @@ tasks.register("InitBackend") { tfInit.mustRunAfter(indexCreate) tfApplyBackend.mustRunAfter(tfInit) uploadLearningMaterials.mustRunAfter(tfApplyBackend) - } tasks.register("DestroyBackend") { From 69526264cef17dfa050787966e328b69e7f06154 Mon Sep 17 00:00:00 2001 From: Timur Sultanov Date: Thu, 4 May 2023 17:26:42 +0400 Subject: [PATCH 2/3] Fix initialization-time execution of tasks which should only be executed fruing build time for Playground containers build --- playground/backend/containers/go/build.gradle | 62 ++++++------ .../backend/containers/java/build.gradle | 36 +++---- .../backend/containers/python/build.gradle | 36 +++---- .../backend/containers/router/build.gradle | 94 ++++++++++--------- .../backend/containers/scio/build.gradle | 28 +++--- 5 files changed, 133 insertions(+), 123 deletions(-) diff --git a/playground/backend/containers/go/build.gradle b/playground/backend/containers/go/build.gradle index 079d6f7aa4e8..ee9d82faa5b0 100644 --- a/playground/backend/containers/go/build.gradle +++ b/playground/backend/containers/go/build.gradle @@ -40,38 +40,40 @@ task copyKafkaEmulator(type: Copy) { into 'build/kafka-emulator/' } -task copyDockerfileDependencies(type: Copy) { +task copyDockerfileDependencies { dependsOn("copyKafkaEmulator") - copy { - from '../../../backend/' - into 'build/src' - exclude 'containers' - } - copy { - from 'entrypoint.sh' - into 'build/' - } - copy { - from 'setup_sdk.sh' - into 'build/' - } - copy { - from '../../../playground' - into 'build/playground' - } - copy { - from '../../../../sdks/go' - into 'build/beam/go' - exclude '**/.gogradle/**' - } - copy { - from '../../../../sdks/go.mod' - into 'build/beam' - } - copy { - from '../../../../sdks/go.sum' - into 'build/beam' + doLast { + copy { + from '../../../backend/' + into 'build/src' + exclude 'containers' + } + copy { + from 'entrypoint.sh' + into 'build/' + } + copy { + from 'setup_sdk.sh' + into 'build/' + } + copy { + from '../../../playground' + into 'build/playground' + } + copy { + from '../../../../sdks/go' + into 'build/beam/go' + exclude '**/.gogradle/**' + } + copy { + from '../../../../sdks/go.mod' + into 'build/beam' + } + copy { + from '../../../../sdks/go.sum' + into 'build/beam' + } } } diff --git a/playground/backend/containers/java/build.gradle b/playground/backend/containers/java/build.gradle index 5b236a9d1b43..f46224be40ff 100644 --- a/playground/backend/containers/java/build.gradle +++ b/playground/backend/containers/java/build.gradle @@ -41,25 +41,27 @@ task copyKafkaEmulator(type: Copy) { into 'build/kafka-emulator/' } -task copyDockerfileDependencies(type: Copy) { +task copyDockerfileDependencies { dependsOn("copyKafkaEmulator") - copy { - from '../../../backend/' - into 'build/src' - exclude 'containers' - } - copy { - from 'entrypoint.sh' - into 'build/' - } - copy { - from '../../../playground' - into 'build/playground' - } - copy { - from '../../../../learning/katas/java' - into 'build/katas' + doLast { + copy { + from '../../../backend/' + into 'build/src' + exclude 'containers' + } + copy { + from 'entrypoint.sh' + into 'build/' + } + copy { + from '../../../playground' + into 'build/playground' + } + copy { + from '../../../../learning/katas/java' + into 'build/katas' + } } } diff --git a/playground/backend/containers/python/build.gradle b/playground/backend/containers/python/build.gradle index 95e1162c828e..30ead18f8618 100644 --- a/playground/backend/containers/python/build.gradle +++ b/playground/backend/containers/python/build.gradle @@ -40,25 +40,27 @@ task copyKafkaEmulator(type: Copy) { into 'build/kafka-emulator/' } -task copyDockerfileDependencies(type: Copy) { +task copyDockerfileDependencies { dependsOn("copyKafkaEmulator") - copy { - from '../../../backend/' - into 'build/src' - exclude 'containers' - } - copy { - from 'entrypoint.sh' - into 'build/' - } - copy { - from '../../../playground' - into 'build/playground' - } - copy { - from '../../../../learning/katas/python' - into 'build/katas' + doLast { + copy { + from '../../../backend/' + into 'build/src' + exclude 'containers' + } + copy { + from 'entrypoint.sh' + into 'build/' + } + copy { + from '../../../playground' + into 'build/playground' + } + copy { + from '../../../../learning/katas/python' + into 'build/katas' + } } } diff --git a/playground/backend/containers/router/build.gradle b/playground/backend/containers/router/build.gradle index 1dc9af6dd9f8..ad5b5a6b1c49 100644 --- a/playground/backend/containers/router/build.gradle +++ b/playground/backend/containers/router/build.gradle @@ -27,67 +27,69 @@ def playgroundJobServerProject = "${project.path.replace('-container', '')}" description = project(playgroundJobServerProject).description + " :: Container" configurations { - dockerDependency + dockerDependency } dependencies { - dockerDependency project(path: playgroundJobServerProject, configuration: "shadow") + dockerDependency project(path: playgroundJobServerProject, configuration: "shadow") } -task copyDockerfileDependencies(type: Copy) { - copy { - from '../../../backend/' - into 'build/src' - exclude 'containers' - } - copy { - from 'entrypoint.sh' - into 'build/' - } - copy { - from '../../../playground' - into 'build/playground' - } - copy { - from '../../../sdks.yaml' - into 'build/' - } - copy { - from '../../../sdks-emulator.yaml' - into 'build/' +task copyDockerfileDependencies { + doLast { + copy { + from '../../../backend/' + into 'build/src' + exclude 'containers' + } + copy { + from 'entrypoint.sh' + into 'build/' + } + copy { + from '../../../playground' + into 'build/playground' + } + copy { + from '../../../sdks.yaml' + into 'build/' + } + copy { + from '../../../sdks-emulator.yaml' + into 'build/' + } } } docker { - name containerImageName( - name: project.docker_image_default_repo_prefix + "playground-backend-router", - root: project.rootProject.hasProperty(["docker-repository-root"]) ? - project.rootProject["docker-repository-root"] : - project.docker_image_default_repo_root) - files "./build/" - tags containerImageTags() - buildArgs(['BASE_IMAGE': project.rootProject.hasProperty(["base-image"]) ? - project.rootProject["base-image"] : - "golang:1.18-bullseye", - 'GIT_COMMIT': getGitCommitHash(), - 'GIT_TIMESTAMP': getGitCommitTimestamp()]) + name containerImageName( + name: project.docker_image_default_repo_prefix + "playground-backend-router", + root: project.rootProject.hasProperty(["docker-repository-root"]) ? + project.rootProject["docker-repository-root"] : + project.docker_image_default_repo_root) + files "./build/" + tags containerImageTags() + buildArgs(['BASE_IMAGE' : project.rootProject.hasProperty(["base-image"]) ? + project.rootProject["base-image"] : + "golang:1.18-bullseye", + 'GIT_COMMIT' : getGitCommitHash(), + 'GIT_TIMESTAMP': getGitCommitTimestamp()]) } task dockerComposeLocalUp() { - dependsOn ':playground:backend:containers:router:docker' - doLast { - exec { - commandLine "docker-compose", "-f", "docker-compose.local.yml", "up", "-d" - } - } + dependsOn ':playground:backend:containers:router:docker' + doLast { + exec { + commandLine "docker-compose", "-f", "docker-compose.local.yml", "up", "-d" + } + } } task dockerComposeLocalDown() { - doLast { - exec { - commandLine "docker-compose", "-f", "docker-compose.local.yml", "down" - } - } + doLast { + exec { + commandLine "docker-compose", "-f", "docker-compose.local.yml", "down" + } + } } // Ensure that we build the required resources and copy and file dependencies from related projects diff --git a/playground/backend/containers/scio/build.gradle b/playground/backend/containers/scio/build.gradle index 145642b4a76b..e4afb763992b 100644 --- a/playground/backend/containers/scio/build.gradle +++ b/playground/backend/containers/scio/build.gradle @@ -40,21 +40,23 @@ task copyKafkaEmulator(type: Copy) { into 'build/kafka-emulator/' } -task copyDockerfileDependencies(type: Copy) { +task copyDockerfileDependencies { dependsOn("copyKafkaEmulator") - copy { - from '../../../backend/' - into 'build/src' - exclude 'containers' - } - copy { - from 'entrypoint.sh' - into 'build/' - } - copy { - from '../../../playground' - into 'build/playground' + doLast { + copy { + from '../../../backend/' + into 'build/src' + exclude 'containers' + } + copy { + from 'entrypoint.sh' + into 'build/' + } + copy { + from '../../../playground' + into 'build/playground' + } } } From a93f5eaaed8f93970d84a624502cc19f02fd5129 Mon Sep 17 00:00:00 2001 From: Timur Sultanov Date: Thu, 4 May 2023 17:46:14 +0400 Subject: [PATCH 3/3] Use dependsOn to specify dependency graph --- .../tour-of-beam/terraform/build.gradle.kts | 207 +++++++++--------- 1 file changed, 101 insertions(+), 106 deletions(-) diff --git a/learning/tour-of-beam/terraform/build.gradle.kts b/learning/tour-of-beam/terraform/build.gradle.kts index 054ce826ec29..05cee8104bf0 100644 --- a/learning/tour-of-beam/terraform/build.gradle.kts +++ b/learning/tour-of-beam/terraform/build.gradle.kts @@ -30,14 +30,15 @@ terraformPlugin { /* init Infrastructure for migrate */ tasks.register("terraformInit") { - // exec args can be passed by commandline, for example + dependsOn("indexcreate") + args( "init", "-migrate-state", "-backend-config=./state.tfbackend" ) } -/* refresh Infrastucture for remote state */ +/* refresh Infrastructure for remote state */ tasks.register("terraformRef") { args( "refresh", @@ -48,29 +49,30 @@ tasks.register("terraformRef") { tasks.register("terraformApplyBackend") { group = "backend-deploy" - var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String + dependsOn("terraformInit") + + val pgRouterHost = project.extensions.extraProperties["pg_router_host"] as String args( "apply", "-auto-approve", "-lock=false", "-parallelism=3", - "-var=pg_router_host=$pg_router_host", + "-var=pg_router_host=$pgRouterHost", "-var=gcloud_init_account=$(gcloud config get-value core/account)", "-var=project_id=$(gcloud config get-value project)", "-var-file=./common.tfvars" ) - - tasks.getByName("uploadLearningMaterials").mustRunAfter(this) - } tasks.register("terraformDestroy") { - var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String + dependsOn("getRouterHost") + + val pgRouterHost = project.extensions.extraProperties["pg_router_host"] as String args( "destroy", "-auto-approve", "-lock=false", - "-var=pg_router_host=$pg_router_host", + "-var=pg_router_host=$pgRouterHost", "-var=gcloud_init_account=$(gcloud config get-value core/account)", "-var=project_id=$(gcloud config get-value project)", "-var-file=./common.tfvars" @@ -78,8 +80,8 @@ tasks.register("terraformDestroy") { } tasks.register("getRouterHost") { + group = "backend-deploy" doLast { - group = "backend-deploy" val result = ByteArrayOutputStream() exec { commandLine( @@ -93,14 +95,17 @@ tasks.register("getRouterHost") { ) standardOutput = result } - val pg_router_host = result.toString().trim().replace("'", "") - project.extensions.extraProperties["pg_router_host"] = pg_router_host + val pgRouterHost = result.toString().trim().replace("'", "") + project.extensions.extraProperties["pg_router_host"] = pgRouterHost } } tasks.register("indexcreate") { + group = "backend-deploy" + dependsOn("getRouterHost") + doLast { - group = "backend-deploy" + val indexpath = "../backend/internal/storage/index.yaml" exec { executable("gcloud") @@ -110,51 +115,55 @@ tasks.register("indexcreate") { } tasks.register("firebaseProjectCreate") { + group = "frontend-deploy" + dependsOn("prepareFirebasercConfig") + doLast { - group = "frontend-deploy" val result = ByteArrayOutputStream() - var project_id = project.property("project_id") as String + val projectId = project.property("project_id") as String exec { executable("firebase") args("projects:list") standardOutput = result } val output = result.toString().trim() - if (output.contains(project_id)) { - println("Firebase is already added to project $project_id.") + if (output.contains(projectId)) { + println("Firebase is already added to project $projectId.") } else { exec { executable("firebase") - args("projects:addfirebase", project_id) + args("projects:addfirebase", projectId) }.assertNormalExitValue() - println("Firebase has been added to project $project_id.") + println("Firebase has been added to project $projectId.") } } } tasks.register("firebaseWebAppCreate") { + group = "frontend-deploy" + dependsOn("firebaseProjectCreate") + doLast { - group = "frontend-deploy" val result = ByteArrayOutputStream() - var project_id = project.property("project_id") as String - var webapp_id = project.property("webapp_id") as String + val projectId = project.property("project_id") as String + val webappId = project.property("webapp_id") as String exec { executable("firebase") - args("apps:list", "--project", project_id) + args("apps:list", "--project", projectId) standardOutput = result } println(result) val output = result.toString() - if (output.contains(webapp_id)) { - println("Webapp id $webapp_id is already created on the project: $project_id.") - val regex = Regex("$webapp_id[│ ]+([\\w:]+)[│ ]+WEB[│ ]+") + if (output.contains(webappId)) { + println("Webapp id $webappId is already created on the project: $projectId.") + val regex = Regex("$webappId[│ ]+([\\w:]+)[│ ]+WEB[│ ]+") val firebaseAppId = regex.find(output)?.groupValues?.get(1)?.trim() project.extensions.extraProperties["firebaseAppId"] = firebaseAppId } else { val result2 = ByteArrayOutputStream() exec { executable("firebase") - args("apps:create", "WEB", webapp_id, "--project", project_id) + args("apps:create", "WEB", webappId, "--project", projectId) standardOutput = result2 }.assertNormalExitValue() val firebaseAppId = @@ -167,8 +176,10 @@ tasks.register("firebaseWebAppCreate") { // firebase apps:sdkconfig WEB AppId tasks.register("getSdkConfigWebApp") { + group = "frontend-deploy" + dependsOn("firebaseWebAppCreate") + doLast { - group = "frontend-deploy" val firebaseAppId = project.extensions.extraProperties["firebaseAppId"] as String val result = ByteArrayOutputStream() exec { @@ -192,8 +203,10 @@ tasks.register("getSdkConfigWebApp") { } tasks.register("prepareFirebaseOptionsDart") { + group = "frontend-deploy" + dependsOn("getSdkConfigWebApp") + doLast { - group = "frontend-deploy" val firebaseConfigData = project.extensions.extraProperties["firebaseConfigData"] as String val file = project.file("../frontend/lib/firebase_options.dart") val content = file.readText() @@ -216,6 +229,8 @@ tasks.register("flutterPubGetPG") { } tasks.register("flutterPubRunPG") { + dependsOn("flutterPubGetPG") + doLast { exec { executable("flutter") @@ -226,6 +241,9 @@ tasks.register("flutterPubRunPG") { } tasks.register("flutterPubGetTob") { + dependsOn("flutterPubRunPG") + dependsOn("getSdkConfigWebApp") + doLast { exec { executable("flutter") @@ -236,6 +254,8 @@ tasks.register("flutterPubGetTob") { } tasks.register("flutterPubRunTob") { + dependsOn("flutterPubGetTob") + doLast { exec { executable("flutter") @@ -246,6 +266,8 @@ tasks.register("flutterPubRunTob") { } tasks.register("flutterBuildWeb") { + dependsOn("flutterPubRunTob") + doLast { exec { executable("flutter") @@ -256,22 +278,25 @@ tasks.register("flutterBuildWeb") { } tasks.register("firebaseDeploy") { + dependsOn("flutterBuildWeb") + doLast { - var project_id = project.property("project_id") as String + val projectId = project.property("project_id") as String exec { - commandLine("firebase", "deploy", "--project", project_id) + commandLine("firebase", "deploy", "--project", projectId) workingDir("../frontend") } } } tasks.register("prepareConfig") { + group = "frontend-deploy" + doLast { - group = "frontend-deploy" - var region = project.property("region") as String - var project_id = project.property("project_id") as String - var environment = project.property("project_environment") as String - var dns_name = project.property("dns-name") as String + val region = project.property("region") as String + val projectId = project.property("project_id") as String + val environment = project.property("project_environment") as String + val dnsName = project.property("dns-name") as String val configFileName = "config.dart" val modulePath = project(":learning:tour-of-beam:frontend").projectDir.absolutePath val file = File("$modulePath/lib/$configFileName") @@ -279,32 +304,34 @@ tasks.register("prepareConfig") { file.writeText( """ const _cloudFunctionsProjectRegion = '$region'; -const _cloudFunctionsProjectId = '$project_id'; +const _cloudFunctionsProjectId = '$projectId'; const cloudFunctionsBaseUrl = 'https://' - '$region-$project_id' + '$region-$projectId' '.cloudfunctions.net/${environment}_'; const String kAnalyticsUA = 'UA-73650088-2'; const String kApiClientURL = -'https://router.${dns_name}'; +'https://router.${dnsName}'; const String kApiJavaClientURL = -'https://java.${dns_name}'; +'https://java.${dnsName}'; const String kApiGoClientURL = -'https://go.${dns_name}'; +'https://go.${dnsName}'; const String kApiPythonClientURL = -'https://python.${dns_name}'; +'https://python.${dnsName}'; const String kApiScioClientURL = -'https://scio.${dns_name}'; +'https://scio.${dnsName}'; """ ) } } tasks.register("prepareFirebasercConfig") { + group = "frontend-deploy" + dependsOn("prepareConfig") + doLast { - group = "frontend-deploy" - var project_id = project.property("project_id") as String + val projectId = project.property("project_id") as String val configFileName = ".firebaserc" val modulePath = project(":learning:tour-of-beam:frontend").projectDir.absolutePath val file = File("$modulePath/$configFileName") @@ -313,7 +340,7 @@ tasks.register("prepareFirebasercConfig") { """ { "projects": { - "default": "$project_id" + "default": "$projectId" } } """ @@ -322,84 +349,52 @@ tasks.register("prepareFirebasercConfig") { } tasks.register("uploadLearningMaterials") { + group = "backend-deploy" + dependsOn("terraformApplyBackend") + doLast { - var project_id = project.property("project_id") as String - group = "backend-deploy" + val projectId = project.property("project_id") as String exec { commandLine("go", "run", "cmd/ci_cd/ci_cd.go") - environment("DATASTORE_PROJECT_ID", project_id) - environment("GOOGLE_PROJECT_ID", project_id) + environment("DATASTORE_PROJECT_ID", projectId) + environment("GOOGLE_PROJECT_ID", projectId) environment("TOB_LEARNING_ROOT", "../learning-content/") workingDir("../backend") } } - dependsOn("terraformApplyBackend") - mustRunAfter("terraformApplyBackend") } /* Tour of Beam backend init */ tasks.register("InitBackend") { group = "backend-deploy" - val getRouterHost = tasks.getByName("getRouterHost") - val indexCreate = tasks.getByName("indexcreate") - val tfInit = tasks.getByName("terraformInit") - val tfApplyBackend = tasks.getByName("terraformApplyBackend") - val uploadLearningMaterials = tasks.getByName("uploadLearningMaterials") - dependsOn(getRouterHost) - dependsOn(indexCreate) - dependsOn(tfInit) - dependsOn(tfApplyBackend) - dependsOn(uploadLearningMaterials) - indexCreate.mustRunAfter(getRouterHost) - tfInit.mustRunAfter(indexCreate) - tfApplyBackend.mustRunAfter(tfInit) - uploadLearningMaterials.mustRunAfter(tfApplyBackend) + + dependsOn("getRouterHost") + dependsOn("indexcreate") + dependsOn("terraformInit") + dependsOn("terraformApplyBackend") + dependsOn("uploadLearningMaterials") } tasks.register("DestroyBackend") { group = "backend-destroy" - val getRouterHost = tasks.getByName("getRouterHost") - val terraformDestroy = tasks.getByName("terraformDestroy") - dependsOn(getRouterHost) - dependsOn(terraformDestroy) - terraformDestroy.mustRunAfter(getRouterHost) + + dependsOn("getRouterHost") + dependsOn("terraformDestroy") } tasks.register("InitFrontend") { group = "frontend-deploy" - val prepareConfig = tasks.getByName("prepareConfig") - val prepareFirebasercConfig = tasks.getByName("prepareFirebasercConfig") - val firebaseProjectCreate = tasks.getByName("firebaseProjectCreate") - val firebaseWebAppCreate = tasks.getByName("firebaseWebAppCreate") - val getSdkConfigWebApp = tasks.getByName("getSdkConfigWebApp") - val prepareFirebaseOptionsDart = tasks.getByName("prepareFirebaseOptionsDart") - val flutterPubGetPG = tasks.getByName("flutterPubGetPG") - val flutterPubRunPG = tasks.getByName("flutterPubRunPG") - val flutterPubGetTob = tasks.getByName("flutterPubGetTob") - val flutterPubRunTob = tasks.getByName("flutterPubRunTob") - val flutterBuildWeb = tasks.getByName("flutterBuildWeb") - val firebaseDeploy = tasks.getByName("firebaseDeploy") - dependsOn(prepareConfig) - dependsOn(prepareFirebasercConfig) - dependsOn(firebaseProjectCreate) - dependsOn(firebaseWebAppCreate) - dependsOn(getSdkConfigWebApp) - dependsOn(prepareFirebaseOptionsDart) - dependsOn(flutterPubGetPG) - dependsOn(flutterPubRunPG) - dependsOn(flutterPubGetTob) - dependsOn(flutterPubRunTob) - dependsOn(flutterBuildWeb) - dependsOn(firebaseDeploy) - prepareFirebasercConfig.mustRunAfter(prepareConfig) - firebaseProjectCreate.mustRunAfter(prepareFirebasercConfig) - firebaseWebAppCreate.mustRunAfter(firebaseProjectCreate) - getSdkConfigWebApp.mustRunAfter(firebaseWebAppCreate) - prepareFirebaseOptionsDart.mustRunAfter(getSdkConfigWebApp) - flutterPubGetPG.mustRunAfter(prepareFirebaseOptionsDart) - flutterPubRunPG.mustRunAfter(flutterPubGetPG) - flutterPubGetTob.mustRunAfter(flutterPubRunPG) - flutterPubRunTob.mustRunAfter(flutterPubGetTob) - flutterBuildWeb.mustRunAfter(flutterPubRunTob) - firebaseDeploy.mustRunAfter(flutterBuildWeb) + + dependsOn("prepareConfig") + dependsOn("prepareFirebasercConfig") + dependsOn("firebaseProjectCreate") + dependsOn("firebaseWebAppCreate") + dependsOn("getSdkConfigWebApp") + dependsOn("prepareFirebaseOptionsDart") + dependsOn("flutterPubGetPG") + dependsOn("flutterPubRunPG") + dependsOn("flutterPubGetTob") + dependsOn("flutterPubRunTob") + dependsOn("flutterBuildWeb") + dependsOn("firebaseDeploy") }