From aec248a5eafb65c205060c138bca6688818d56b3 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Tue, 28 May 2024 13:47:28 +0200 Subject: [PATCH 01/38] First IaC test --- .../workflows/build-test-scan-push-images.yml | 52 +++++++++++- iac/kubernetes.tf | 33 ++++++++ iac/main.tf | 45 ++++++++++ iac/network.tf | 64 +++++++++++++++ iac/nsg.tf | 23 ++++++ iac/peering.tf | 24 ++++++ iac/rg.tf | 82 +++++++++++++++++++ iac/tailscale_cloudinit.tpl | 13 +++ iac/tailscale_operator.tf | 24 ++++++ iac/variables.tf | 34 ++++++++ iac/versions.tf | 18 ++++ iac/vm.tf | 74 +++++++++++++++++ 12 files changed, 485 insertions(+), 1 deletion(-) create mode 100644 iac/kubernetes.tf create mode 100644 iac/main.tf create mode 100644 iac/network.tf create mode 100644 iac/nsg.tf create mode 100644 iac/peering.tf create mode 100644 iac/rg.tf create mode 100644 iac/tailscale_cloudinit.tpl create mode 100644 iac/tailscale_operator.tf create mode 100644 iac/variables.tf create mode 100644 iac/versions.tf create mode 100644 iac/vm.tf diff --git a/.github/workflows/build-test-scan-push-images.yml b/.github/workflows/build-test-scan-push-images.yml index af4bde9..42afb8a 100644 --- a/.github/workflows/build-test-scan-push-images.yml +++ b/.github/workflows/build-test-scan-push-images.yml @@ -120,4 +120,54 @@ jobs: uses: github/codeql-action/upload-sarif@v2 with: sarif_file: 'trivy-results.sarif' - category: ${{ matrix.image.name }} \ No newline at end of file + category: ${{ matrix.image.name }} + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + submodules: true + + - name: Install Terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: latest + + - name: Set WorkDir + run: echo "##[set-output name=workdir;]${{ github.workspace }}/bootstrap" + + - name: Login to Azure + run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + - name: DC specific variables overwrite + run: | + echo "copying datacenter specific variables" + cp -v ${{ github.workspace }}/global_variables/*_variables.tf ${{ steps.set_workdir.outputs.workdir }}/. + continue-on-error: true + + - name: Tokenize terraform file + uses: qetza/replacetokens@v3 + with: + root-directory: ${{ steps.set_workdir.outputs.workdir }} + target-files: '*.tf' + write-bom: false + action-on-missing: 'fail' + + - name: Terraform Apply + run: | + cd ${{ steps.set_workdir.outputs.workdir }} + export ARM_CLIENT_SECRET=${{ secrets.CLIENT_SECRET }} + export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} + export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} + export TF_VAR_tenant_id=${{ secrets.AZURERM_TENANT_ID }} + export TF_VAR_tailscale_auth_key=$(secrets.TAILSCALE_AUTH_KEY) + export TF_VAR_tailscale_client_id=$(secrets.TAILSCALE_CLIENT_ID) + export TF_VAR_tailscale_client_secret=$(secrets.TAILSCALE_CLIENT_SECRET) + export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " + terraform init + terraform plan -out=tfplan.bin -input=false + terraform apply -auto-approve "tfplan.bin" + + - name: Logout of Azure + run: az logout \ No newline at end of file diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf new file mode 100644 index 0000000..d8fd5af --- /dev/null +++ b/iac/kubernetes.tf @@ -0,0 +1,33 @@ +resource "azurerm_kubernetes_cluster" "testCluster" { + name = "testCluster" + location = data.azurerm_resource_group.rgruntime.location + resource_group_name = data.azurerm_resource_group.rgruntime.name + dns_prefix = "testCluster" + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + } + + network_profile { + network_plugin = "azure" + load_balancer_sku = "standard" + outbound_type = "loadBalancer" + } + + identity { + type = "SystemAssigned" + } + + private_cluster_enabled = true +} + +/* +output "client_certificate" { + value = azurerm_kubernetes_cluster.testCluster.kube_config.0.client_certificate +} + +output "kube_config" { + value = azurerm_kubernetes_cluster.testCluster.kube_config_raw +}*/ \ No newline at end of file diff --git a/iac/main.tf b/iac/main.tf new file mode 100644 index 0000000..f0aa45f --- /dev/null +++ b/iac/main.tf @@ -0,0 +1,45 @@ + +variable "subscription_id" { + type = string +} +variable "tenant_id" { + type = string +} + + + +provider "azurerm" { + subscription_id = var.subscription_id + tenant_id = var.tenant_id + features { + key_vault { + purge_soft_delete_on_destroy = true + recover_soft_deleted_key_vaults = true + } + } +} + +provider "helm" { + kubernetes { + host = azurerm_kubernetes_cluster.testCluster.kube_config.0.host + client_certificate = base64decode(azurerm_kubernetes_cluster.testCluster.kube_config.0.client_certificate) + client_key = base64decode(azurerm_kubernetes_cluster.testCluster.kube_config.0.client_key) + cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.testCluster.kube_config.0.cluster_ca_certificate) + } +} + +terraform { + backend "azurerm" { + use_azuread_auth = true + } +} + + +locals { + common_tags = { + release = "HandsOnCloudNative" + purpose = "class" + classification = "sensitive" + central = "yes" + } +} \ No newline at end of file diff --git a/iac/network.tf b/iac/network.tf new file mode 100644 index 0000000..ffc2981 --- /dev/null +++ b/iac/network.tf @@ -0,0 +1,64 @@ +# a first network : vnets are larger network grouping, typically segmented via CIDR ranges +# high level network design should be done on paper and widely communicated +resource "azurerm_virtual_network" "vnet-student" { + name = "vnet-student" + address_space = ["10.1.0.0/16"] + location = var.globals.location + resource_group_name = data.azurerm_resource_group.rgruntime.name +} + +# a second network +# resource "azurerm_virtual_network" "vnet-platform" { +# name = "vnet-platform" +# address_space = ["10.2.0.0/16"] +# location = var.globals.location +# resource_group_name = data.azurerm_resource_group.rgruntime.name +# } + +# subnets +resource "azurerm_subnet" "snet-student-vm" { + name = "snet-student-vm" + resource_group_name = data.azurerm_resource_group.rgruntime.name + virtual_network_name = "vnet-student" + depends_on = [ azurerm_virtual_network.vnet-student ] + address_prefixes = ["10.1.1.0/24"] +} +# # subnets +# # A second subnet to demo peering the two so they can talk to each other +# resource "azurerm_subnet" "snet-buildagents-k8s" { +# name = "snet-buildagents-k8s" +# resource_group_name = data.azurerm_resource_group.rgruntime.name +# virtual_network_name = "vnet-buildagents" +# depends_on = [ azurerm_virtual_network.vnet-buildagents ] +# address_prefixes = ["10.1.2.0/24"] +# } + +# security groups need to be attached +# in Azure, they attach to subnets (in Openstack they attach to VMs and layer2 ports) +resource "azurerm_subnet_network_security_group_association" "student-vm" { + subnet_id = azurerm_subnet.snet-student-vm.id + network_security_group_id = azurerm_network_security_group.student.id +} + +######################################## ROLE ASSIGNEMENTS ################################## +# In general, if we want terraform to assign roles to itself or other objects -> we need to be careful +# this can easily be used for privilegdge escalation +# on the other hand: if you can protect your buildagents well as well as make sure the IaC branches are safe +# it is best practise to let automation assign or generally handle as much of these fine grained settings as possible +# it is easier to not make mistakes if automation handles this for you +# + +# # give the current terraform agent the permission to Read (in order to read the network config) +# resource "azurerm_role_assignment" "networkread" { +# scope = "/subscriptions/${var.subscription_id}/resourceGroups/rg-service-not2day" +# role_definition_name = "Reader" +# principal_id = data.azurerm_client_config.current.object_id +# depends_on = [ azurerm_virtual_network.vnet-buildagents ] +# } + +#resource "azurerm_role_assignment" "vmcontrib" { +# scope = "/subscriptions/${var.subscription_id}/resourceGroups/rg-service-not2day" +# role_definition_name = "Virtual Machine Contributor" +# principal_id = data.azurerm_client_config.current.object_id +# depends_on = [ azurerm_virtual_network.vnet-buildagents ] +#} \ No newline at end of file diff --git a/iac/nsg.tf b/iac/nsg.tf new file mode 100644 index 0000000..41de0e6 --- /dev/null +++ b/iac/nsg.tf @@ -0,0 +1,23 @@ + +resource azurerm_network_security_group "student" { + name = "student-network-security-group" + location = var.globals.location + resource_group_name = data.azurerm_resource_group.rgruntime.name + + #tags = local.common_tags +} + +resource "azurerm_network_security_rule" "lab_nsg" { + name = "Tailscale" + description = "Tailscale UDP port for direct connections. Reduces latency." + priority = 1010 + direction = "Inbound" + access = "Allow" + protocol = "Udp" + source_port_range = "*" + destination_port_range = 41641 + source_address_prefix = "*" + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.rgruntime.name + network_security_group_name = azurerm_network_security_group.student.name +} \ No newline at end of file diff --git a/iac/peering.tf b/iac/peering.tf new file mode 100644 index 0000000..ff89931 --- /dev/null +++ b/iac/peering.tf @@ -0,0 +1,24 @@ + +## IF you have two Vnets and you want them to talk to each other, this is a bridge +## It cuts through your network segmentation, so again: be intentional about it please + +# resource "azurerm_virtual_network_peering" "vnet-peering-agents" { +# name = "agents_to_platform" +# resource_group_name = data.azurerm_resource_group.rgruntime.name +# virtual_network_name = azurerm_virtual_network.vnet-buildagents.name +# remote_virtual_network_id = azurerm_virtual_network.vnet-platform.id +# allow_virtual_network_access = true +# allow_forwarded_traffic = true + +# } + + +# resource "azurerm_virtual_network_peering" "vnet-peering-platform" { +# name = "platform_to_agents" +# resource_group_name = data.azurerm_resource_group.rgruntime.name +# virtual_network_name = azurerm_virtual_network.vnet-platform.name +# remote_virtual_network_id = azurerm_virtual_network.vnet-buildagents.id +# allow_virtual_network_access = true +# allow_forwarded_traffic = true + +# } diff --git a/iac/rg.tf b/iac/rg.tf new file mode 100644 index 0000000..70c5da1 --- /dev/null +++ b/iac/rg.tf @@ -0,0 +1,82 @@ + + +data "azurerm_client_config" "current" {} + + +#Read my target RG + +data "azurerm_resource_group" "rgruntime" { + name = "rg-service-not2day" + +} + +################################################################ +# IF you are generating any secrets, you need to put them somewhere +# most ideally, you put them into a keyvault of the same lifecycle-stage as the asset the key belongs to +################################################################## + + + +resource "azurerm_key_vault" "kvservice" { + name = "kv-service-not2day-2" + location = data.azurerm_resource_group.rgruntime.location + resource_group_name = data.azurerm_resource_group.rgruntime.name + enabled_for_disk_encryption = true + tenant_id = data.azurerm_client_config.current.tenant_id + soft_delete_retention_days = 7 + purge_protection_enabled = false + + sku_name = "standard" + + + network_acls { + ip_rules = ["0.0.0.0/0" ] #change this + default_action= "Deny" + bypass = "AzureServices" + } + + + access_policy { + tenant_id = data.azurerm_client_config.current.tenant_id + ## Students you must look up your Users Object id in Entra ID and put it here + object_id = var.myuser + + + + secret_permissions = [ + "Get", + "List", + "Restore", + "Delete", + "Set", + "Recover", + "Backup", + ] + + + } + # We are giving this SP access over the current vault + access_policy { + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azurerm_client_config.current.object_id + + + + secret_permissions = [ + "Get", + "List", + "Restore", + "Delete", + "Set", + "Recover", + "Backup", + ] + + + } + + +tags = local.common_tags + +} + diff --git a/iac/tailscale_cloudinit.tpl b/iac/tailscale_cloudinit.tpl new file mode 100644 index 0000000..8667e46 --- /dev/null +++ b/iac/tailscale_cloudinit.tpl @@ -0,0 +1,13 @@ +#cloud-config +apt: + sources: + tailscale.list: + source: deb https://pkgs.tailscale.com/stable/ubuntu focal main + keyid: 2596A99EAAB33821893C0A79458CA832957F5868 +packages: + - tailscale +runcmd: + - "tailscale up -authkey ${tailscale_auth_key} --advertise-routes=10.1.1.0/24,168.63.129.16/32 --accept-dns=false" + - "echo 'net.ipv4.ip_forward = 1' | sudo tee -a /etc/sysctl.conf" + - "echo 'net.ipv6.conf.all.forwarding = 1' | sudo tee -a /etc/sysctl.conf" + - "sysctl -p /etc/sysctl.conf" \ No newline at end of file diff --git a/iac/tailscale_operator.tf b/iac/tailscale_operator.tf new file mode 100644 index 0000000..e692726 --- /dev/null +++ b/iac/tailscale_operator.tf @@ -0,0 +1,24 @@ +resource "helm_release" "tailscale_operator" { + name = "tailscale-operator" + repository = "https://pkgs.tailscale.com/helmcharts" + chart = "tailscale-operator" + namespace = "tailscale" + + set { + name = "oauth.clientId" + value = var.tailscale_client_id // Replace with your actual Tailscale client ID + } + + set { + name = "oauth.clientSecret" + value = var.tailscale_client_secret // Replace with your actual Tailscale client secret + } + + set { + name = "apiServerProxyConfig.mode" + value = "true" + } + + create_namespace = true + wait = true +} \ No newline at end of file diff --git a/iac/variables.tf b/iac/variables.tf new file mode 100644 index 0000000..242619a --- /dev/null +++ b/iac/variables.tf @@ -0,0 +1,34 @@ + + +variable "globals" { + type = map(any) + + default = { + location = "West Europe" + } +} +variable "network" { + type = map(any) + + # most Austrian University VPNs + default = { + allowlist_ips = "128.130.0.0/15,193.171.80.0/21,193.170.16.0/20,193.170.185.0/24,129.27.0.0/16,138.232.0.0/16,141.244.0.0/16,143.50.0.0/16,143.205.0.0/16,140.78.0.0/16,193.186.176.0/22,193.186.172.0/22,149.148.0.0/16,192.82.158.0/24,193.171.96.0/21,193.171.104.0/22" + } + +} + +variable "myuser"{ + default = "56ea78b9-6d9f-495b-85ac-7caa86ccc191" +} + +variable "tailscale_auth_key" { + type = string +} + +variable "tailscale_client_secret" { + type = string +} + +variable "tailscale_client_id" { + type = string +} \ No newline at end of file diff --git a/iac/versions.tf b/iac/versions.tf new file mode 100644 index 0000000..4175fb1 --- /dev/null +++ b/iac/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_providers { + + azuread = { + source = "hashicorp/azuread" + version = ">= 2.7.0" + } + azurerm = { + source = "hashicorp/azurerm" + version = ">= 2.59.0" + } + tls = { + source = "hashicorp/tls" + version = "4.0.4" + } + } + required_version = ">= 0.13" +} diff --git a/iac/vm.tf b/iac/vm.tf new file mode 100644 index 0000000..0e3a18d --- /dev/null +++ b/iac/vm.tf @@ -0,0 +1,74 @@ + +resource "azurerm_network_interface" "student" { + name = "nic-student" + location = data.azurerm_resource_group.rgruntime.location + resource_group_name = data.azurerm_resource_group.rgruntime.name + + ip_configuration { + name = "internal" + subnet_id = azurerm_subnet.snet-student-vm.id + private_ip_address_allocation = "Dynamic" + } +} + +resource "tls_private_key" "example_ssh" { + algorithm = "RSA" + rsa_bits = 4096 +} + +## Commented out to save money , VMs cost real money! Dont leave them on if you dont need them + +/* + +resource "azurerm_linux_virtual_machine" "vm" { + name = "vm" + resource_group_name = data.azurerm_resource_group.rgruntime.name + location = data.azurerm_resource_group.rgruntime.location + size = "Standard_F1" + priority = "Spot" + eviction_policy = "Deallocate" + disable_password_authentication = "true" + admin_username = "adminusercrcr" + network_interface_ids = [ + azurerm_network_interface.student.id, + ] + + admin_ssh_key { + username = "adminusercrcr" + public_key = tls_private_key.example_ssh.public_key_openssh + } + + os_disk { + caching = "ReadWrite" + storage_account_type = "Standard_LRS" + } + + source_image_reference { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts" + version = "latest" + } + + custom_data = base64encode(templatefile("${path.module}/tailscale_cloudinit.tpl", { + tailscale_auth_key = var.tailscale_auth_key + })) +} + + +resource "azurerm_key_vault_secret" "publicsshkey" { + name = "student-ssh-key-public" + value = tls_private_key.example_ssh.public_key_openssh + key_vault_id = azurerm_key_vault.kvservice.id + tags = local.common_tags + depends_on = [azurerm_key_vault.kvservice] + +} +resource "azurerm_key_vault_secret" "sshkey" { + name = "student-ssh-key-private" + value = tls_private_key.example_ssh.private_key_openssh + key_vault_id = azurerm_key_vault.kvservice.id + tags = local.common_tags + depends_on = [azurerm_key_vault.kvservice] + +}*/ \ No newline at end of file From 3e9e956777b6b63bd672d8f6c76e59b7105d4be3 Mon Sep 17 00:00:00 2001 From: Jonas Konrad <24313231+jalaka@users.noreply.github.com> Date: Tue, 28 May 2024 13:52:37 +0200 Subject: [PATCH 02/38] test --- .github/workflows/build-test-scan-push-images.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/build-test-scan-push-images.yml b/.github/workflows/build-test-scan-push-images.yml index 42afb8a..dd479b3 100644 --- a/.github/workflows/build-test-scan-push-images.yml +++ b/.github/workflows/build-test-scan-push-images.yml @@ -7,6 +7,7 @@ on: tags: - v*.*.* + - feature/aks-ci pull_request: branches: @@ -146,13 +147,6 @@ jobs: cp -v ${{ github.workspace }}/global_variables/*_variables.tf ${{ steps.set_workdir.outputs.workdir }}/. continue-on-error: true - - name: Tokenize terraform file - uses: qetza/replacetokens@v3 - with: - root-directory: ${{ steps.set_workdir.outputs.workdir }} - target-files: '*.tf' - write-bom: false - action-on-missing: 'fail' - name: Terraform Apply run: | From 34b0f8fc248792f15716d0ad3a44bc1af8f018b1 Mon Sep 17 00:00:00 2001 From: Jonas Konrad <24313231+jalaka@users.noreply.github.com> Date: Tue, 28 May 2024 13:54:59 +0200 Subject: [PATCH 03/38] test2 --- .github/workflows/build-test-scan-push-images.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-test-scan-push-images.yml b/.github/workflows/build-test-scan-push-images.yml index dd479b3..1f7ad13 100644 --- a/.github/workflows/build-test-scan-push-images.yml +++ b/.github/workflows/build-test-scan-push-images.yml @@ -141,12 +141,6 @@ jobs: - name: Login to Azure run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} - - name: DC specific variables overwrite - run: | - echo "copying datacenter specific variables" - cp -v ${{ github.workspace }}/global_variables/*_variables.tf ${{ steps.set_workdir.outputs.workdir }}/. - continue-on-error: true - - name: Terraform Apply run: | @@ -155,9 +149,9 @@ jobs: export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} export TF_VAR_tenant_id=${{ secrets.AZURERM_TENANT_ID }} - export TF_VAR_tailscale_auth_key=$(secrets.TAILSCALE_AUTH_KEY) - export TF_VAR_tailscale_client_id=$(secrets.TAILSCALE_CLIENT_ID) - export TF_VAR_tailscale_client_secret=$(secrets.TAILSCALE_CLIENT_SECRET) + export TF_VAR_tailscale_auth_key=${{secrets.TAILSCALE_AUTH_KEY}} + export TF_VAR_tailscale_client_id=${{secrets.TAILSCALE_CLIENT_ID}} + export TF_VAR_tailscale_client_secret=${{secrets.TAILSCALE_CLIENT_SECRET}} export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " terraform init terraform plan -out=tfplan.bin -input=false From 1d8deaa39145cc7d705efafae4e49a7d95c2ca2d Mon Sep 17 00:00:00 2001 From: Jonas Konrad <24313231+jalaka@users.noreply.github.com> Date: Tue, 28 May 2024 13:59:12 +0200 Subject: [PATCH 04/38] change dir --- .github/workflows/build-test-scan-push-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-scan-push-images.yml b/.github/workflows/build-test-scan-push-images.yml index 1f7ad13..a49009f 100644 --- a/.github/workflows/build-test-scan-push-images.yml +++ b/.github/workflows/build-test-scan-push-images.yml @@ -136,7 +136,7 @@ jobs: terraform_version: latest - name: Set WorkDir - run: echo "##[set-output name=workdir;]${{ github.workspace }}/bootstrap" + run: echo "##[set-output name=workdir;]${{ github.workspace }}/iac" - name: Login to Azure run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} From d0b5bd2d4af3e383f1bfb9215571800908500957 Mon Sep 17 00:00:00 2001 From: Jonas Konrad <24313231+jalaka@users.noreply.github.com> Date: Tue, 28 May 2024 14:02:33 +0200 Subject: [PATCH 05/38] workdir --- .github/workflows/build-test-scan-push-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-scan-push-images.yml b/.github/workflows/build-test-scan-push-images.yml index a49009f..2d9e8de 100644 --- a/.github/workflows/build-test-scan-push-images.yml +++ b/.github/workflows/build-test-scan-push-images.yml @@ -144,7 +144,7 @@ jobs: - name: Terraform Apply run: | - cd ${{ steps.set_workdir.outputs.workdir }} + cd ${{ github.workspace }}/iac" export ARM_CLIENT_SECRET=${{ secrets.CLIENT_SECRET }} export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} From d9a911dd21d9fea7a6a6df8f7eac9361ff06af3a Mon Sep 17 00:00:00 2001 From: Jonas Konrad <24313231+jalaka@users.noreply.github.com> Date: Tue, 28 May 2024 14:03:54 +0200 Subject: [PATCH 06/38] typo --- .github/workflows/build-test-scan-push-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-scan-push-images.yml b/.github/workflows/build-test-scan-push-images.yml index 2d9e8de..ee990a2 100644 --- a/.github/workflows/build-test-scan-push-images.yml +++ b/.github/workflows/build-test-scan-push-images.yml @@ -144,7 +144,7 @@ jobs: - name: Terraform Apply run: | - cd ${{ github.workspace }}/iac" + cd ${{ github.workspace }}/iac export ARM_CLIENT_SECRET=${{ secrets.CLIENT_SECRET }} export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} From df6e3b90267cf2eda8bb5763695082c122d764a8 Mon Sep 17 00:00:00 2001 From: Jonas Konrad <24313231+jalaka@users.noreply.github.com> Date: Tue, 28 May 2024 14:16:41 +0200 Subject: [PATCH 07/38] Change kv name --- iac/rg.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iac/rg.tf b/iac/rg.tf index 70c5da1..7494082 100644 --- a/iac/rg.tf +++ b/iac/rg.tf @@ -18,7 +18,7 @@ data "azurerm_resource_group" "rgruntime" { resource "azurerm_key_vault" "kvservice" { - name = "kv-service-not2day-2" + name = "kv-service-not2day-3" location = data.azurerm_resource_group.rgruntime.location resource_group_name = data.azurerm_resource_group.rgruntime.name enabled_for_disk_encryption = true From 4b66d710a555fc4c29b159239c4f8a8e207b8bf1 Mon Sep 17 00:00:00 2001 From: Jonas Konrad <24313231+jalaka@users.noreply.github.com> Date: Tue, 28 May 2024 14:32:29 +0200 Subject: [PATCH 08/38] Add working operator --- .github/workflows/build-test-scan-push-images.yml | 1 + ...ilscale_operator.tf => tailscale_operator.tf.bak_not_working} | 0 2 files changed, 1 insertion(+) rename iac/{tailscale_operator.tf => tailscale_operator.tf.bak_not_working} (100%) diff --git a/.github/workflows/build-test-scan-push-images.yml b/.github/workflows/build-test-scan-push-images.yml index ee990a2..5c44fe9 100644 --- a/.github/workflows/build-test-scan-push-images.yml +++ b/.github/workflows/build-test-scan-push-images.yml @@ -156,6 +156,7 @@ jobs: terraform init terraform plan -out=tfplan.bin -input=false terraform apply -auto-approve "tfplan.bin" + az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TF_VAR_tailscale_client_id}} --set-string oauth.clientSecret=${{secrets.TF_VAR_tailscale_client_secret}} --set-string apiServerProxyConfig.mode=true --wait" - name: Logout of Azure run: az logout \ No newline at end of file diff --git a/iac/tailscale_operator.tf b/iac/tailscale_operator.tf.bak_not_working similarity index 100% rename from iac/tailscale_operator.tf rename to iac/tailscale_operator.tf.bak_not_working From 8a3f8bc996217d90f89072dedd5b518f1fb58010 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 3 Jun 2024 01:26:05 +0200 Subject: [PATCH 09/38] Add destroy ci/cd option --- .../workflows/build-test-scan-push-images.yml | 40 +-------- .github/workflows/deploy-az.yml | 90 +++++++++++++++++++ 2 files changed, 91 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/deploy-az.yml diff --git a/.github/workflows/build-test-scan-push-images.yml b/.github/workflows/build-test-scan-push-images.yml index 5c44fe9..1625597 100644 --- a/.github/workflows/build-test-scan-push-images.yml +++ b/.github/workflows/build-test-scan-push-images.yml @@ -121,42 +121,4 @@ jobs: uses: github/codeql-action/upload-sarif@v2 with: sarif_file: 'trivy-results.sarif' - category: ${{ matrix.image.name }} - deploy: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - submodules: true - - - name: Install Terraform - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: latest - - - name: Set WorkDir - run: echo "##[set-output name=workdir;]${{ github.workspace }}/iac" - - - name: Login to Azure - run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} - - - - name: Terraform Apply - run: | - cd ${{ github.workspace }}/iac - export ARM_CLIENT_SECRET=${{ secrets.CLIENT_SECRET }} - export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} - export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} - export TF_VAR_tenant_id=${{ secrets.AZURERM_TENANT_ID }} - export TF_VAR_tailscale_auth_key=${{secrets.TAILSCALE_AUTH_KEY}} - export TF_VAR_tailscale_client_id=${{secrets.TAILSCALE_CLIENT_ID}} - export TF_VAR_tailscale_client_secret=${{secrets.TAILSCALE_CLIENT_SECRET}} - export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " - terraform init - terraform plan -out=tfplan.bin -input=false - terraform apply -auto-approve "tfplan.bin" - az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TF_VAR_tailscale_client_id}} --set-string oauth.clientSecret=${{secrets.TF_VAR_tailscale_client_secret}} --set-string apiServerProxyConfig.mode=true --wait" - - - name: Logout of Azure - run: az logout \ No newline at end of file + category: ${{ matrix.image.name }} \ No newline at end of file diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml new file mode 100644 index 0000000..c378bd1 --- /dev/null +++ b/.github/workflows/deploy-az.yml @@ -0,0 +1,90 @@ +name: Deploy to azure + +on: + workflow_dispatch: + inputs: + purge: + description: 'Delete az infrastructure' + required: false + default: false + type: boolean +env: + REGISTRY: ghcr.io + NAMESPACE: austriandatalab + SUB_NAMESPACE: indiegamestream +jobs: + deploy: + runs-on: ubuntu-latest + if: ${{ ! github.event.inputs.purge }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: latest + + + - name: Login to Azure + run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + + - name: Terraform Apply + working-directory: ./iac + run: | + export ARM_CLIENT_SECRET=${{ secrets.CLIENT_SECRET }} + export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} + export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} + export TF_VAR_tenant_id=${{ secrets.AZURERM_TENANT_ID }} + export TF_VAR_tailscale_auth_key=${{secrets.TAILSCALE_AUTH_KEY}} + export TF_VAR_tailscale_client_id=${{secrets.TAILSCALE_CLIENT_ID}} + export TF_VAR_tailscale_client_secret=${{secrets.TAILSCALE_CLIENT_SECRET}} + export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " + terraform init + terraform plan -out=tfplan.bin -input=false + terraform apply -auto-approve "tfplan.bin" + az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TF_VAR_tailscale_client_id}} --set-string oauth.clientSecret=${{secrets.TF_VAR_tailscale_client_secret}} --set-string apiServerProxyConfig.mode=true --wait" + + - name: Logout of Azure + run: az logout + purge: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.purge }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: latest + + + - name: Login to Azure + run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + + - name: Terraform Apply + working-directory: ./iac + run: | + export ARM_CLIENT_SECRET=${{ secrets.CLIENT_SECRET }} + export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} + export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} + export TF_VAR_tenant_id=${{ secrets.AZURERM_TENANT_ID }} + export TF_VAR_tailscale_auth_key=${{secrets.TAILSCALE_AUTH_KEY}} + export TF_VAR_tailscale_client_id=${{secrets.TAILSCALE_CLIENT_ID}} + export TF_VAR_tailscale_client_secret=${{secrets.TAILSCALE_CLIENT_SECRET}} + export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " + terraform init + terraform plan -out=tfplan.bin -input=false + terraform apply -auto-approve "tfplan.bin" -destroy + terraform apply -auto-approve "tfplan.bin" -destroy + terraform apply -auto-approve "tfplan.bin" -destroy + + - name: Logout of Azure + run: az logout \ No newline at end of file From 411c9f759986627d7ac1982924465a93c1b99c5b Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 3 Jun 2024 01:35:23 +0200 Subject: [PATCH 10/38] Fix workflows syntax --- .github/workflows/deploy-az.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index c378bd1..5da188a 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -8,6 +8,7 @@ on: required: false default: false type: boolean + env: REGISTRY: ghcr.io NAMESPACE: austriandatalab From a44a18c36201b8d924f0d04941613ee2d82ccf6d Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 3 Jun 2024 01:39:11 +0200 Subject: [PATCH 11/38] Add push trigger to fix github --- .github/workflows/deploy-az.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 5da188a..f6c1fac 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -1,6 +1,8 @@ name: Deploy to azure on: + push: + workflow_dispatch: inputs: purge: @@ -8,7 +10,7 @@ on: required: false default: false type: boolean - + env: REGISTRY: ghcr.io NAMESPACE: austriandatalab From 51bff2a59782b0a4d3e417d13510bb647f8f4946 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 3 Jun 2024 01:45:18 +0200 Subject: [PATCH 12/38] Change terraform destroy command --- .github/workflows/deploy-az.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index f6c1fac..95d2589 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -2,7 +2,7 @@ name: Deploy to azure on: push: - + workflow_dispatch: inputs: purge: @@ -85,9 +85,9 @@ jobs: export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " terraform init terraform plan -out=tfplan.bin -input=false - terraform apply -auto-approve "tfplan.bin" -destroy - terraform apply -auto-approve "tfplan.bin" -destroy - terraform apply -auto-approve "tfplan.bin" -destroy + terraform destroy -auto-approve "tfplan.bin" + terraform destroy -auto-approve "tfplan.bin" + terraform destroy -auto-approve "tfplan.bin" - name: Logout of Azure run: az logout \ No newline at end of file From 10d7b3c1768db03495b2f97b7dd0ab9158cbbb3d Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 3 Jun 2024 02:05:43 +0200 Subject: [PATCH 13/38] Change terraform destroy command --- .github/workflows/deploy-az.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 95d2589..595a843 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -1,8 +1,6 @@ name: Deploy to azure on: - push: - workflow_dispatch: inputs: purge: @@ -85,9 +83,9 @@ jobs: export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " terraform init terraform plan -out=tfplan.bin -input=false - terraform destroy -auto-approve "tfplan.bin" - terraform destroy -auto-approve "tfplan.bin" - terraform destroy -auto-approve "tfplan.bin" + terraform destroy + terraform destroy + terraform destroy - name: Logout of Azure run: az logout \ No newline at end of file From 9e3b37e4b9d27c63255d852e57a9a44c0b21defe Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 3 Jun 2024 02:07:50 +0200 Subject: [PATCH 14/38] Remove confirmation for terraform --- .github/workflows/deploy-az.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 595a843..1e23129 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -83,9 +83,9 @@ jobs: export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " terraform init terraform plan -out=tfplan.bin -input=false - terraform destroy - terraform destroy - terraform destroy + terraform destroy -auto-approve + terraform destroy -auto-approve + terraform destroy -auto-approve - name: Logout of Azure run: az logout \ No newline at end of file From 726844c5a959c47c3df81719c10fec19309dfbc8 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 3 Jun 2024 02:48:49 +0200 Subject: [PATCH 15/38] Optimize vm price --- iac/kubernetes.tf | 4 ++-- iac/network.tf | 10 +++++++--- iac/vm.tf | 4 +++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index d8fd5af..44da0bf 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -7,12 +7,12 @@ resource "azurerm_kubernetes_cluster" "testCluster" { default_node_pool { name = "default" node_count = 1 - vm_size = "Standard_DS2_v2" + vm_size = "Standard_B2ms" } network_profile { network_plugin = "azure" - load_balancer_sku = "standard" + load_balancer_sku = "basic" outbound_type = "loadBalancer" } diff --git a/iac/network.tf b/iac/network.tf index ffc2981..2e265e6 100644 --- a/iac/network.tf +++ b/iac/network.tf @@ -1,11 +1,13 @@ # a first network : vnets are larger network grouping, typically segmented via CIDR ranges # high level network design should be done on paper and widely communicated + +/* resource "azurerm_virtual_network" "vnet-student" { name = "vnet-student" address_space = ["10.1.0.0/16"] location = var.globals.location resource_group_name = data.azurerm_resource_group.rgruntime.name -} +}*/ # a second network # resource "azurerm_virtual_network" "vnet-platform" { @@ -16,13 +18,14 @@ resource "azurerm_virtual_network" "vnet-student" { # } # subnets +/* resource "azurerm_subnet" "snet-student-vm" { name = "snet-student-vm" resource_group_name = data.azurerm_resource_group.rgruntime.name virtual_network_name = "vnet-student" depends_on = [ azurerm_virtual_network.vnet-student ] address_prefixes = ["10.1.1.0/24"] -} +}*/ # # subnets # # A second subnet to demo peering the two so they can talk to each other # resource "azurerm_subnet" "snet-buildagents-k8s" { @@ -35,10 +38,11 @@ resource "azurerm_subnet" "snet-student-vm" { # security groups need to be attached # in Azure, they attach to subnets (in Openstack they attach to VMs and layer2 ports) +/* resource "azurerm_subnet_network_security_group_association" "student-vm" { subnet_id = azurerm_subnet.snet-student-vm.id network_security_group_id = azurerm_network_security_group.student.id -} +}*/ ######################################## ROLE ASSIGNEMENTS ################################## # In general, if we want terraform to assign roles to itself or other objects -> we need to be careful diff --git a/iac/vm.tf b/iac/vm.tf index 0e3a18d..01b2748 100644 --- a/iac/vm.tf +++ b/iac/vm.tf @@ -1,4 +1,6 @@ +/* + resource "azurerm_network_interface" "student" { name = "nic-student" location = data.azurerm_resource_group.rgruntime.location @@ -18,7 +20,7 @@ resource "tls_private_key" "example_ssh" { ## Commented out to save money , VMs cost real money! Dont leave them on if you dont need them -/* + resource "azurerm_linux_virtual_machine" "vm" { name = "vm" From 159d6e5c62debaab6191cf194b25c5afed36606e Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 3 Jun 2024 02:59:01 +0200 Subject: [PATCH 16/38] Workaround for github service bug --- .github/workflows/deploy-az.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 1e23129..0ef9e15 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -6,7 +6,6 @@ on: purge: description: 'Delete az infrastructure' required: false - default: false type: boolean env: @@ -16,7 +15,7 @@ env: jobs: deploy: runs-on: ubuntu-latest - if: ${{ ! github.event.inputs.purge }} + if: ${{ github.event.inputs.purge == 'false' }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -53,7 +52,7 @@ jobs: run: az logout purge: runs-on: ubuntu-latest - if: ${{ github.event.inputs.purge }} + if: ${{ github.event.inputs.purge == 'true' }} steps: - name: Checkout repository uses: actions/checkout@v4 From 5d7ce11ca340bf02be447a8b39d44e60111d212a Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 3 Jun 2024 03:02:33 +0200 Subject: [PATCH 17/38] Sitch to private cluster --- iac/kubernetes.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index 44da0bf..87f1dac 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -12,7 +12,7 @@ resource "azurerm_kubernetes_cluster" "testCluster" { network_profile { network_plugin = "azure" - load_balancer_sku = "basic" + load_balancer_sku = "standard" outbound_type = "loadBalancer" } From 09600f43eb75598cfc5672bb22d4859c1c8beb67 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Fri, 14 Jun 2024 16:42:04 +0200 Subject: [PATCH 18/38] Change gh secrets for tailscale operator --- .github/workflows/deploy-az.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 0ef9e15..1ac20e7 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -39,15 +39,14 @@ jobs: export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} export TF_VAR_tenant_id=${{ secrets.AZURERM_TENANT_ID }} - export TF_VAR_tailscale_auth_key=${{secrets.TAILSCALE_AUTH_KEY}} - export TF_VAR_tailscale_client_id=${{secrets.TAILSCALE_CLIENT_ID}} - export TF_VAR_tailscale_client_secret=${{secrets.TAILSCALE_CLIENT_SECRET}} export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " terraform init terraform plan -out=tfplan.bin -input=false terraform apply -auto-approve "tfplan.bin" - az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TF_VAR_tailscale_client_id}} --set-string oauth.clientSecret=${{secrets.TF_VAR_tailscale_client_secret}} --set-string apiServerProxyConfig.mode=true --wait" - + - name: Apply tailscale operator + working-directory: ./iac + run: | + az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait" - name: Logout of Azure run: az logout purge: @@ -76,9 +75,6 @@ jobs: export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} export TF_VAR_tenant_id=${{ secrets.AZURERM_TENANT_ID }} - export TF_VAR_tailscale_auth_key=${{secrets.TAILSCALE_AUTH_KEY}} - export TF_VAR_tailscale_client_id=${{secrets.TAILSCALE_CLIENT_ID}} - export TF_VAR_tailscale_client_secret=${{secrets.TAILSCALE_CLIENT_SECRET}} export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " terraform init terraform plan -out=tfplan.bin -input=false From 8b8cc89bc558b7a108061a5a1ac6ce7874975809 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Tue, 28 May 2024 16:07:12 +0200 Subject: [PATCH 19/38] Add rbac to operator --- operator/config/manager/kustomization.yaml | 6 +++ operator/config/rbac/role.yaml | 45 +++++++++++++++++++ .../controller/stream/game_controller.go | 4 ++ 3 files changed, 55 insertions(+) diff --git a/operator/config/manager/kustomization.yaml b/operator/config/manager/kustomization.yaml index 5c5f0b8..ad13e96 100644 --- a/operator/config/manager/kustomization.yaml +++ b/operator/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: controller + newTag: latest diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index 3b9b22f..2e3d502 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -4,6 +4,39 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - stream.indiegamestream.com resources: @@ -30,6 +63,18 @@ rules: - get - patch - update +- apiGroups: + - stunner.l7mp.io + resources: + - gatewayconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - stunner.l7mp.io resources: diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index 409d154..a6e70c6 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -47,6 +47,10 @@ type GameReconciler struct { //+kubebuilder:rbac:groups=stream.indiegamestream.com,resources=games/status,verbs=get;update;patch //+kubebuilder:rbac:groups=stream.indiegamestream.com,resources=games/finalizers,verbs=update //+kubebuilder:rbac:groups=stunner.l7mp.io,resources=udproutes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=stunner.l7mp.io,resources=gatewayconfigs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create; // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From bd9a2bff715c211732d853df90a2236b0a2b11c4 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Fri, 14 Jun 2024 14:41:24 +0200 Subject: [PATCH 20/38] Merge changes --- operator/internal/controller/stream/game_controller.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index a6e70c6..ffa97d5 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -501,6 +501,9 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r } func (r *GameReconciler) constructLoadBalancer(game *streamv1.Game, name string, selector string, port int32) (*corev1.Service, error) { + + //className := "tailscale" + svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -508,6 +511,7 @@ func (r *GameReconciler) constructLoadBalancer(game *streamv1.Game, name string, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{"app": selector}, + // LoadBalancerClass: &className, Ports: []corev1.ServicePort{ { Port: port, From a1657b53ade328033541f9611db2cffa8f70bde5 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Fri, 14 Jun 2024 16:45:11 +0200 Subject: [PATCH 21/38] Delete unused terraform files --- iac/tailscale_cloudinit.tpl | 13 ------------ iac/tailscale_operator.tf.bak_not_working | 24 ----------------------- iac/variables.tf | 12 ------------ 3 files changed, 49 deletions(-) delete mode 100644 iac/tailscale_cloudinit.tpl delete mode 100644 iac/tailscale_operator.tf.bak_not_working diff --git a/iac/tailscale_cloudinit.tpl b/iac/tailscale_cloudinit.tpl deleted file mode 100644 index 8667e46..0000000 --- a/iac/tailscale_cloudinit.tpl +++ /dev/null @@ -1,13 +0,0 @@ -#cloud-config -apt: - sources: - tailscale.list: - source: deb https://pkgs.tailscale.com/stable/ubuntu focal main - keyid: 2596A99EAAB33821893C0A79458CA832957F5868 -packages: - - tailscale -runcmd: - - "tailscale up -authkey ${tailscale_auth_key} --advertise-routes=10.1.1.0/24,168.63.129.16/32 --accept-dns=false" - - "echo 'net.ipv4.ip_forward = 1' | sudo tee -a /etc/sysctl.conf" - - "echo 'net.ipv6.conf.all.forwarding = 1' | sudo tee -a /etc/sysctl.conf" - - "sysctl -p /etc/sysctl.conf" \ No newline at end of file diff --git a/iac/tailscale_operator.tf.bak_not_working b/iac/tailscale_operator.tf.bak_not_working deleted file mode 100644 index e692726..0000000 --- a/iac/tailscale_operator.tf.bak_not_working +++ /dev/null @@ -1,24 +0,0 @@ -resource "helm_release" "tailscale_operator" { - name = "tailscale-operator" - repository = "https://pkgs.tailscale.com/helmcharts" - chart = "tailscale-operator" - namespace = "tailscale" - - set { - name = "oauth.clientId" - value = var.tailscale_client_id // Replace with your actual Tailscale client ID - } - - set { - name = "oauth.clientSecret" - value = var.tailscale_client_secret // Replace with your actual Tailscale client secret - } - - set { - name = "apiServerProxyConfig.mode" - value = "true" - } - - create_namespace = true - wait = true -} \ No newline at end of file diff --git a/iac/variables.tf b/iac/variables.tf index 242619a..1e7d9ba 100644 --- a/iac/variables.tf +++ b/iac/variables.tf @@ -19,16 +19,4 @@ variable "network" { variable "myuser"{ default = "56ea78b9-6d9f-495b-85ac-7caa86ccc191" -} - -variable "tailscale_auth_key" { - type = string -} - -variable "tailscale_client_secret" { - type = string -} - -variable "tailscale_client_id" { - type = string } \ No newline at end of file From 85f095e9f9bade5621e9251784d9b8407afe8d1a Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Fri, 14 Jun 2024 17:37:41 +0200 Subject: [PATCH 22/38] Add uninstall for tailscale operator --- .github/workflows/deploy-az.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 1ac20e7..05fa410 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -46,6 +46,7 @@ jobs: - name: Apply tailscale operator working-directory: ./iac run: | + az aks command invoke -n testCluster -g rg-service-not2day --command "helm uninstall tailscale-operator --namespace=tailscale || true" az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait" - name: Logout of Azure run: az logout From 4979b01ff0ef9e1dc52a9dc81abec02caa2bab46 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Fri, 14 Jun 2024 18:05:04 +0200 Subject: [PATCH 23/38] Add stunner operator --- .github/workflows/deploy-az.yml | 6 ++++++ iac/kubernetes.tf | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 05fa410..0f5bd83 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -48,6 +48,12 @@ jobs: run: | az aks command invoke -n testCluster -g rg-service-not2day --command "helm uninstall tailscale-operator --namespace=tailscale || true" az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait" + - name: Apply stunner operator + working-directory: ./helm/stunner + run: | + az aks command invoke -n testCluster -g rg-service-not2day --command "helm uninstall stunner --namespace=stunner || true" + az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add stunner https://l7mp.io/stunner && helm repo update && helm dependency build . --skip-refresh && helm install stunner . --create-namespace --namespace=stunner" + - name: Logout of Azure run: az logout purge: diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index 87f1dac..ddc5eed 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -8,6 +8,11 @@ resource "azurerm_kubernetes_cluster" "testCluster" { name = "default" node_count = 1 vm_size = "Standard_B2ms" + upgrade_settings { + drain_timeout_in_minutes = 0 + max_surge = "10%" + node_soak_duration_in_minutes = 0 + } } network_profile { From fa7535bd50a4df2c96f963979dd093158d516b05 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Fri, 14 Jun 2024 18:28:56 +0200 Subject: [PATCH 24/38] Add tailscale kubernetes config to github-ci --- .github/workflows/deploy-az.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 0f5bd83..f38ee24 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -48,11 +48,15 @@ jobs: run: | az aks command invoke -n testCluster -g rg-service-not2day --command "helm uninstall tailscale-operator --namespace=tailscale || true" az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait" - - name: Apply stunner operator - working-directory: ./helm/stunner - run: | - az aks command invoke -n testCluster -g rg-service-not2day --command "helm uninstall stunner --namespace=stunner || true" - az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add stunner https://l7mp.io/stunner && helm repo update && helm dependency build . --skip-refresh && helm install stunner . --create-namespace --namespace=stunner" + - name: Connect to tailscale + uses: tailscale/github-action@v2 + with: + oauth-client-id: ${{secrets.TAILSCALE_CLIENT_ID}} + oauth-secret: ${{secrets.TAILSCALE_CLIENT_SECRET}} + tags: tag:ci + - name: Configure kubernetes config + run: tailscale configure kubeconfig tailscale-operator + - name: Logout of Azure run: az logout From 75c07d9006d421c2ffa0bc3d9bd1525ea8bff404 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Fri, 14 Jun 2024 18:38:42 +0200 Subject: [PATCH 25/38] Use different secret for client --- .github/workflows/deploy-az.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index f38ee24..e9a160f 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -51,8 +51,8 @@ jobs: - name: Connect to tailscale uses: tailscale/github-action@v2 with: - oauth-client-id: ${{secrets.TAILSCALE_CLIENT_ID}} - oauth-secret: ${{secrets.TAILSCALE_CLIENT_SECRET}} + oauth-client-id: ${{secrets.TAILSCALE_CLIENT_ID_2}} + oauth-secret: ${{secrets.TAILSCALE_CLIENT_SECRET_2}} tags: tag:ci - name: Configure kubernetes config run: tailscale configure kubeconfig tailscale-operator From bfaca6b99ae61b2524821010e223f2d4e2b59a13 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Fri, 14 Jun 2024 18:46:25 +0200 Subject: [PATCH 26/38] Add kubectl to github ci --- .github/workflows/deploy-az.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index e9a160f..b0b297c 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -27,6 +27,18 @@ jobs: with: terraform_version: latest + - name: Install Helm + uses: azure/setup-helm@v4.2.0 + with: + version: 'latest' + id: install1 + + - name: Install kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'latest' + id: install2 + - name: Login to Azure run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} @@ -57,6 +69,9 @@ jobs: - name: Configure kubernetes config run: tailscale configure kubeconfig tailscale-operator + - name: Check working cluster + run: kubectl get pods -A + - name: Logout of Azure run: az logout From 092ca62a8884aa309fb5f77fa7e0dde98006fe96 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Fri, 14 Jun 2024 18:56:51 +0200 Subject: [PATCH 27/38] Add operator deployment --- .github/workflows/deploy-az.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index b0b297c..0ef9c05 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -72,7 +72,20 @@ jobs: - name: Check working cluster run: kubectl get pods -A + - name: Install stunner + working-directory: ./scripts/localenv + run: make install_stunner + + - name: Install game operator manifests + working-directory: ./operator + run: make install + + - name: Deploy game operator + working-directory: ./operator + run: make deploy IMG=ghcr.io/austriandatalab/indiegamestream/operator:sha-36404cc8de274ca24575023405a2f7868dffa610 + + - name: Logout of Azure run: az logout purge: @@ -93,6 +106,8 @@ jobs: - name: Login to Azure run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + - name: Terraform Apply working-directory: ./iac From 8a7ded2f74a0e037bedab096f1825b02038ec7fc Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Fri, 14 Jun 2024 19:21:36 +0200 Subject: [PATCH 28/38] Removed unnecessary Terraform --- iac/network.tf | 68 ------------------------------------------- iac/peering.tf | 24 --------------- iac/variables.tf | 11 ------- iac/vm.tf | 76 ------------------------------------------------ 4 files changed, 179 deletions(-) delete mode 100644 iac/network.tf delete mode 100644 iac/peering.tf delete mode 100644 iac/vm.tf diff --git a/iac/network.tf b/iac/network.tf deleted file mode 100644 index 2e265e6..0000000 --- a/iac/network.tf +++ /dev/null @@ -1,68 +0,0 @@ -# a first network : vnets are larger network grouping, typically segmented via CIDR ranges -# high level network design should be done on paper and widely communicated - -/* -resource "azurerm_virtual_network" "vnet-student" { - name = "vnet-student" - address_space = ["10.1.0.0/16"] - location = var.globals.location - resource_group_name = data.azurerm_resource_group.rgruntime.name -}*/ - -# a second network -# resource "azurerm_virtual_network" "vnet-platform" { -# name = "vnet-platform" -# address_space = ["10.2.0.0/16"] -# location = var.globals.location -# resource_group_name = data.azurerm_resource_group.rgruntime.name -# } - -# subnets -/* -resource "azurerm_subnet" "snet-student-vm" { - name = "snet-student-vm" - resource_group_name = data.azurerm_resource_group.rgruntime.name - virtual_network_name = "vnet-student" - depends_on = [ azurerm_virtual_network.vnet-student ] - address_prefixes = ["10.1.1.0/24"] -}*/ -# # subnets -# # A second subnet to demo peering the two so they can talk to each other -# resource "azurerm_subnet" "snet-buildagents-k8s" { -# name = "snet-buildagents-k8s" -# resource_group_name = data.azurerm_resource_group.rgruntime.name -# virtual_network_name = "vnet-buildagents" -# depends_on = [ azurerm_virtual_network.vnet-buildagents ] -# address_prefixes = ["10.1.2.0/24"] -# } - -# security groups need to be attached -# in Azure, they attach to subnets (in Openstack they attach to VMs and layer2 ports) -/* -resource "azurerm_subnet_network_security_group_association" "student-vm" { - subnet_id = azurerm_subnet.snet-student-vm.id - network_security_group_id = azurerm_network_security_group.student.id -}*/ - -######################################## ROLE ASSIGNEMENTS ################################## -# In general, if we want terraform to assign roles to itself or other objects -> we need to be careful -# this can easily be used for privilegdge escalation -# on the other hand: if you can protect your buildagents well as well as make sure the IaC branches are safe -# it is best practise to let automation assign or generally handle as much of these fine grained settings as possible -# it is easier to not make mistakes if automation handles this for you -# - -# # give the current terraform agent the permission to Read (in order to read the network config) -# resource "azurerm_role_assignment" "networkread" { -# scope = "/subscriptions/${var.subscription_id}/resourceGroups/rg-service-not2day" -# role_definition_name = "Reader" -# principal_id = data.azurerm_client_config.current.object_id -# depends_on = [ azurerm_virtual_network.vnet-buildagents ] -# } - -#resource "azurerm_role_assignment" "vmcontrib" { -# scope = "/subscriptions/${var.subscription_id}/resourceGroups/rg-service-not2day" -# role_definition_name = "Virtual Machine Contributor" -# principal_id = data.azurerm_client_config.current.object_id -# depends_on = [ azurerm_virtual_network.vnet-buildagents ] -#} \ No newline at end of file diff --git a/iac/peering.tf b/iac/peering.tf deleted file mode 100644 index ff89931..0000000 --- a/iac/peering.tf +++ /dev/null @@ -1,24 +0,0 @@ - -## IF you have two Vnets and you want them to talk to each other, this is a bridge -## It cuts through your network segmentation, so again: be intentional about it please - -# resource "azurerm_virtual_network_peering" "vnet-peering-agents" { -# name = "agents_to_platform" -# resource_group_name = data.azurerm_resource_group.rgruntime.name -# virtual_network_name = azurerm_virtual_network.vnet-buildagents.name -# remote_virtual_network_id = azurerm_virtual_network.vnet-platform.id -# allow_virtual_network_access = true -# allow_forwarded_traffic = true - -# } - - -# resource "azurerm_virtual_network_peering" "vnet-peering-platform" { -# name = "platform_to_agents" -# resource_group_name = data.azurerm_resource_group.rgruntime.name -# virtual_network_name = azurerm_virtual_network.vnet-platform.name -# remote_virtual_network_id = azurerm_virtual_network.vnet-buildagents.id -# allow_virtual_network_access = true -# allow_forwarded_traffic = true - -# } diff --git a/iac/variables.tf b/iac/variables.tf index 1e7d9ba..40f7bc9 100644 --- a/iac/variables.tf +++ b/iac/variables.tf @@ -1,5 +1,3 @@ - - variable "globals" { type = map(any) @@ -7,15 +5,6 @@ variable "globals" { location = "West Europe" } } -variable "network" { - type = map(any) - - # most Austrian University VPNs - default = { - allowlist_ips = "128.130.0.0/15,193.171.80.0/21,193.170.16.0/20,193.170.185.0/24,129.27.0.0/16,138.232.0.0/16,141.244.0.0/16,143.50.0.0/16,143.205.0.0/16,140.78.0.0/16,193.186.176.0/22,193.186.172.0/22,149.148.0.0/16,192.82.158.0/24,193.171.96.0/21,193.171.104.0/22" - } - -} variable "myuser"{ default = "56ea78b9-6d9f-495b-85ac-7caa86ccc191" diff --git a/iac/vm.tf b/iac/vm.tf deleted file mode 100644 index 01b2748..0000000 --- a/iac/vm.tf +++ /dev/null @@ -1,76 +0,0 @@ - -/* - -resource "azurerm_network_interface" "student" { - name = "nic-student" - location = data.azurerm_resource_group.rgruntime.location - resource_group_name = data.azurerm_resource_group.rgruntime.name - - ip_configuration { - name = "internal" - subnet_id = azurerm_subnet.snet-student-vm.id - private_ip_address_allocation = "Dynamic" - } -} - -resource "tls_private_key" "example_ssh" { - algorithm = "RSA" - rsa_bits = 4096 -} - -## Commented out to save money , VMs cost real money! Dont leave them on if you dont need them - - - -resource "azurerm_linux_virtual_machine" "vm" { - name = "vm" - resource_group_name = data.azurerm_resource_group.rgruntime.name - location = data.azurerm_resource_group.rgruntime.location - size = "Standard_F1" - priority = "Spot" - eviction_policy = "Deallocate" - disable_password_authentication = "true" - admin_username = "adminusercrcr" - network_interface_ids = [ - azurerm_network_interface.student.id, - ] - - admin_ssh_key { - username = "adminusercrcr" - public_key = tls_private_key.example_ssh.public_key_openssh - } - - os_disk { - caching = "ReadWrite" - storage_account_type = "Standard_LRS" - } - - source_image_reference { - publisher = "Canonical" - offer = "0001-com-ubuntu-server-jammy" - sku = "22_04-lts" - version = "latest" - } - - custom_data = base64encode(templatefile("${path.module}/tailscale_cloudinit.tpl", { - tailscale_auth_key = var.tailscale_auth_key - })) -} - - -resource "azurerm_key_vault_secret" "publicsshkey" { - name = "student-ssh-key-public" - value = tls_private_key.example_ssh.public_key_openssh - key_vault_id = azurerm_key_vault.kvservice.id - tags = local.common_tags - depends_on = [azurerm_key_vault.kvservice] - -} -resource "azurerm_key_vault_secret" "sshkey" { - name = "student-ssh-key-private" - value = tls_private_key.example_ssh.private_key_openssh - key_vault_id = azurerm_key_vault.kvservice.id - tags = local.common_tags - depends_on = [azurerm_key_vault.kvservice] - -}*/ \ No newline at end of file From b6d79a2338c2fa8cb947b86401005088962e9197 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Fri, 14 Jun 2024 19:22:50 +0200 Subject: [PATCH 29/38] Removed feature/aks-ci from tags --- .github/workflows/build-test-scan-push-images.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-test-scan-push-images.yml b/.github/workflows/build-test-scan-push-images.yml index 1625597..af4bde9 100644 --- a/.github/workflows/build-test-scan-push-images.yml +++ b/.github/workflows/build-test-scan-push-images.yml @@ -7,7 +7,6 @@ on: tags: - v*.*.* - - feature/aks-ci pull_request: branches: From 46a3323b80ba911b9d81d76debc0535f33bbb235 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Fri, 14 Jun 2024 21:31:12 +0200 Subject: [PATCH 30/38] Added frontend and API install to deploy workflow --- .github/workflows/deploy-az.yml | 62 +++++++++++++++++++++++++++++++-- helm/mysql/values.yaml | 4 +++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 0ef9c05..d5c7e15 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -12,6 +12,7 @@ env: REGISTRY: ghcr.io NAMESPACE: austriandatalab SUB_NAMESPACE: indiegamestream + LABEL: sha-f641ffb9ebff0a3f8c8f9b968bfd50f83a316370 jobs: deploy: runs-on: ubuntu-latest @@ -72,6 +73,17 @@ jobs: - name: Check working cluster run: kubectl get pods -A + - name: Install MySQL + working-directory: ./helm/mysql + run: | + helm repo add mysql-operator https://mysql.github.io/mysql-operator/ + helm repo update + helm install mysql-operator mysql-operator/mysql-operator --version "2.1.3" --wait \ + --create-namespace --namespace=mysql-operator + helm install mysql mysql-operator/mysql-innodbcluster --version "2.1.3" --wait \ + --create-namespace --namespace=mysql -f values.yaml + --set-string credentials.root.password=${{ secrets.MYSQL_ROOT_PASSWORD }} + - name: Install stunner working-directory: ./scripts/localenv run: make install_stunner @@ -82,9 +94,55 @@ jobs: - name: Deploy game operator working-directory: ./operator - run: make deploy IMG=ghcr.io/austriandatalab/indiegamestream/operator:sha-36404cc8de274ca24575023405a2f7868dffa610 - + run: make deploy IMG=${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.SUB_NAMESPACE }}/operator:${{ env.LABEL }} + - name: Wait for MySQL to be ready + run: | + while true; do + POD_STATUS=$(kubectl get pod mysql-0 -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); + if [ "$POD_STATUS" ]; then + echo "Pod mysql-0 has been created with status: $POD_STATUS"; + break; + else + echo "Waiting for pod mysql-0 to be created..."; + sleep 5; + fi + done + kubectl wait --for=condition=Ready pod/mysql-0 -n mysql --timeout=120s + + while true; do + POD_STATUS=$(kubectl get pod -l app.kubernetes.io/component=router -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); \ + if [ "$POD_STATUS" ]; then + echo "MySQL router has been created with status: $POD_STATUS"; + break; + else + echo "Waiting for MySQL router to be created..."; + sleep 5; + fi + done + kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=router -n mysql --timeout=120s + + - name: Install API + working-directory: ./helm/api + run: | + helm install -f values.yaml \ + --set-string env.mysqlRootPassword=${{ secrets.MYSQL_ROOT_PASSWORD }} \ + --set-string image.label=${{ env.LABEL }} \ + api . + + - name: Wait for external IP of API + run: | + until [ -n "$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do + sleep 5 + done + + - name: Install frontend + working-directory: ./helm/frontend + run: | + helm install -f values.yaml \ + --set-string appConfig.apiUrl=http://$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$(kubectl get svc api -n api -o jsonpath='{.spec.ports[0].port}') \ + --set-string image.label=${{ env.LABEL }} \ + frontend . - name: Logout of Azure run: az logout diff --git a/helm/mysql/values.yaml b/helm/mysql/values.yaml index e69de29..d8bf4cc 100644 --- a/helm/mysql/values.yaml +++ b/helm/mysql/values.yaml @@ -0,0 +1,4 @@ +serverInstances: 1 + +tls: + useSelfSigned: true \ No newline at end of file From 660cfdcf557edc179ba0aa3af7da0bddbd11016c Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Fri, 14 Jun 2024 21:57:43 +0200 Subject: [PATCH 31/38] Added Tailscale annotations to frontend and API services --- helm/api/templates/api-service.yaml | 3 +++ helm/frontend/templates/frontend-service.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/helm/api/templates/api-service.yaml b/helm/api/templates/api-service.yaml index cfd960d..5d43502 100644 --- a/helm/api/templates/api-service.yaml +++ b/helm/api/templates/api-service.yaml @@ -5,6 +5,9 @@ metadata: namespace: {{ .Values.appName }} labels: app: {{ .Values.appName }} + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "api" spec: type: LoadBalancer selector: diff --git a/helm/frontend/templates/frontend-service.yaml b/helm/frontend/templates/frontend-service.yaml index cfd960d..0ee96e9 100644 --- a/helm/frontend/templates/frontend-service.yaml +++ b/helm/frontend/templates/frontend-service.yaml @@ -5,6 +5,9 @@ metadata: namespace: {{ .Values.appName }} labels: app: {{ .Values.appName }} + annotations: + tailscale.com/expose: "true" + tailscale.com/hostname: "frontend" spec: type: LoadBalancer selector: From 6b30c411c7b3780bef1c3bdd27d90360ebc5cfa0 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Fri, 14 Jun 2024 22:24:12 +0200 Subject: [PATCH 32/38] Actually using tailscale LoadBalancer --- helm/api/templates/api-service.yaml | 4 +++- helm/api/values.yaml | 4 ++++ helm/frontend/templates/frontend-service.yaml | 3 +++ helm/frontend/values.yaml | 4 ++++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/helm/api/templates/api-service.yaml b/helm/api/templates/api-service.yaml index 5d43502..f7ed7b1 100644 --- a/helm/api/templates/api-service.yaml +++ b/helm/api/templates/api-service.yaml @@ -6,10 +6,12 @@ metadata: labels: app: {{ .Values.appName }} annotations: - tailscale.com/expose: "true" tailscale.com/hostname: "api" spec: type: LoadBalancer + {{ if .Values.service.spec.loadBalancerClass }} + loadBalancerClass: {{ .Values.service.spec.loadBalancerClass }} + {{ end }} selector: app: {{ .Values.appName }} ports: diff --git a/helm/api/values.yaml b/helm/api/values.yaml index fe4a44a..69e93a1 100644 --- a/helm/api/values.yaml +++ b/helm/api/values.yaml @@ -7,6 +7,10 @@ image: label: v1.0.0 # Change to proper version tag pullPolicy: IfNotPresent +service: + spec: + loadBalancerClass: tailscale + env: ginMode: release mysqlDatabase: api diff --git a/helm/frontend/templates/frontend-service.yaml b/helm/frontend/templates/frontend-service.yaml index 0ee96e9..c762855 100644 --- a/helm/frontend/templates/frontend-service.yaml +++ b/helm/frontend/templates/frontend-service.yaml @@ -10,6 +10,9 @@ metadata: tailscale.com/hostname: "frontend" spec: type: LoadBalancer + {{ if .Values.service.spec.loadBalancerClass }} + loadBalancerClass: {{ .Values.service.spec.loadBalancerClass }} + {{ end }} selector: app: {{ .Values.appName }} ports: diff --git a/helm/frontend/values.yaml b/helm/frontend/values.yaml index a6bc86a..cca23f0 100644 --- a/helm/frontend/values.yaml +++ b/helm/frontend/values.yaml @@ -7,6 +7,10 @@ image: label: v1.0.0 # Change to proper version tag pullPolicy: IfNotPresent +service: + spec: + loadBalancerClass: tailscale + webRootDirectory: /usr/share/nginx/html appConfig: From 32faad1f699dd9ff8c77c7be52747bb388c01fee Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Fri, 14 Jun 2024 22:25:52 +0200 Subject: [PATCH 33/38] loadBalancerClass set to null on local deployment --- helm/api/values-dev.yaml | 4 ++++ helm/frontend/values-dev.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/helm/api/values-dev.yaml b/helm/api/values-dev.yaml index 955540b..5295723 100644 --- a/helm/api/values-dev.yaml +++ b/helm/api/values-dev.yaml @@ -3,6 +3,10 @@ image: label: localenv pullPolicy: Never +service: + spec: + loadBalancerClass: null + env: ginMode: debug mysqlRootPassword: root \ No newline at end of file diff --git a/helm/frontend/values-dev.yaml b/helm/frontend/values-dev.yaml index 0965df8..b62e761 100644 --- a/helm/frontend/values-dev.yaml +++ b/helm/frontend/values-dev.yaml @@ -3,5 +3,9 @@ image: label: localenv pullPolicy: Never +service: + spec: + loadBalancerClass: null + appConfig: production: false \ No newline at end of file From 6668530b6474dd95a2155d24061aed84212e7c35 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Fri, 14 Jun 2024 22:28:04 +0200 Subject: [PATCH 34/38] Remove tailscale.com/expose annotation from frontend service --- helm/frontend/templates/frontend-service.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/helm/frontend/templates/frontend-service.yaml b/helm/frontend/templates/frontend-service.yaml index c762855..2aa6693 100644 --- a/helm/frontend/templates/frontend-service.yaml +++ b/helm/frontend/templates/frontend-service.yaml @@ -6,7 +6,6 @@ metadata: labels: app: {{ .Values.appName }} annotations: - tailscale.com/expose: "true" tailscale.com/hostname: "frontend" spec: type: LoadBalancer From 081945b94fd4c7f7f3f1aa3701336114ab1e6ca9 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Fri, 14 Jun 2024 23:11:30 +0200 Subject: [PATCH 35/38] Added env variables --- .github/workflows/deploy-az.yml | 12 ++++++++++-- helm/api/templates/api-deployment.yaml | 20 ++++++++++++++++++-- iac/kubernetes.tf | 6 +++--- iac/variables.tf | 5 +++++ 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index d5c7e15..3b48973 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -59,8 +59,8 @@ jobs: - name: Apply tailscale operator working-directory: ./iac run: | - az aks command invoke -n testCluster -g rg-service-not2day --command "helm uninstall tailscale-operator --namespace=tailscale || true" - az aks command invoke -n testCluster -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait" + az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm uninstall tailscale-operator --namespace=tailscale || true" + az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait" - name: Connect to tailscale uses: tailscale/github-action@v2 with: @@ -127,6 +127,14 @@ jobs: run: | helm install -f values.yaml \ --set-string env.mysqlRootPassword=${{ secrets.MYSQL_ROOT_PASSWORD }} \ + --set-string env.azureTenantId=${{ secrets.AZURE_TENANT_ID }} + --set-string env.azureClientId=${{ secrets.CLIENT_ID }} + --set-string env.azureClientSecret=${{ secrets.CLIENT_SECRET }} + --set-string env.azureStorageAccount=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }} + --set-string env.azureContainerName=${{ secrets.AZURERM_CONTAINER_NAME }} + --set-string env.azureAksClusterName=${{ secrets.AZURERM_AKS_CLUSTER_NAME }} + --set-string env.azurermSubscriptionId=${{ secrets.AZURERM_SUBSCRIPTION_ID }} + --set-string env.azurermResourceGroupName=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }} --set-string image.label=${{ env.LABEL }} \ api . diff --git a/helm/api/templates/api-deployment.yaml b/helm/api/templates/api-deployment.yaml index 2e12126..741c5fd 100644 --- a/helm/api/templates/api-deployment.yaml +++ b/helm/api/templates/api-deployment.yaml @@ -31,8 +31,24 @@ spec: - name: MYSQL_ROOT_USER value: {{ .Values.env.mysqlRootUser | quote }} - name: MYSQL_ROOT_PASSWORD - value: {{ .Values.env.mysqlRootPassword | quote }} + value: {{ required ".Values.env.mysqlRootPassword is required." .Values.env.mysqlRootPassword | quote }} - name: MYSQL_HOST value: {{ .Values.env.mysqlHost | quote }} - name: MYSQL_PORT - value: {{ .Values.env.mysqlPort | quote }} \ No newline at end of file + value: {{ .Values.env.mysqlPort | quote }} + - name: AZURE_TENANT_ID + value: {{ required ".Values.env.azureTenantId is required." .Values.env.azureTenantId | quote }} + - name: AZURE_CLIENT_ID + value: {{ required ".Values.env.azureClientId is required." .Values.env.azureClientId | quote }} + - name: AZURE_CLIENT_SECRET + value: {{ required ".Values.env.azureClientSecret is required." .Values.env.azureClientSecret | quote }} + - name: AZURE_STORAGE_ACCOUNT + value: {{ required ".Values.env.azureStorageAccount is required." .Values.env.azureStorageAccount | quote }} + - name: AZURE_CONTAINER_NAME + value: {{ required ".Values.env.azureContainerName is required." .Values.env.azureContainerName | quote }} + - name: AZURE_AKS_CLUSTER_NAME + value: {{ required ".Values.env.azureAksClusterName is required." .Values.env.azureAksClusterName | quote }} + - name: AZURERM_SUBSCRIPTION_ID + value: {{ required ".Values.env.azurermSubscriptionId is required." .Values.env.azurermSubscriptionId | quote }} + - name: AZURERM_RESOURCE_GROUP_NAME + value: {{ required ".Values.env.azurermResourceGroupName is required." .Values.env.azurermResourceGroupName | quote }} \ No newline at end of file diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index ddc5eed..d0b6329 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -1,8 +1,8 @@ -resource "azurerm_kubernetes_cluster" "testCluster" { - name = "testCluster" +resource "azurerm_kubernetes_cluster" "indiegamestream-cluster" { + name = var.cluster_name location = data.azurerm_resource_group.rgruntime.location resource_group_name = data.azurerm_resource_group.rgruntime.name - dns_prefix = "testCluster" + dns_prefix = var.cluster_name default_node_pool { name = "default" diff --git a/iac/variables.tf b/iac/variables.tf index 40f7bc9..5fb85ae 100644 --- a/iac/variables.tf +++ b/iac/variables.tf @@ -8,4 +8,9 @@ variable "globals" { variable "myuser"{ default = "56ea78b9-6d9f-495b-85ac-7caa86ccc191" +} + +variable "cluster_name" { + type = string + default = "indiegamestream-cluster" } \ No newline at end of file From 6229457459321ed16aa26124ecaf57ad02faacd7 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Fri, 14 Jun 2024 23:14:53 +0200 Subject: [PATCH 36/38] Changed resource name back to old name --- iac/kubernetes.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index d0b6329..8cdb75b 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -1,4 +1,4 @@ -resource "azurerm_kubernetes_cluster" "indiegamestream-cluster" { +resource "azurerm_kubernetes_cluster" "testCluster" { name = var.cluster_name location = data.azurerm_resource_group.rgruntime.location resource_group_name = data.azurerm_resource_group.rgruntime.name From a2fcb4538a3370f13a1baab179014adce6c03811 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Sat, 15 Jun 2024 12:23:32 +0200 Subject: [PATCH 37/38] Commented out install of API and frontend --- .github/workflows/deploy-az.yml | 128 ++++++++++++++++---------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 3b48973..92f5444 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -73,16 +73,16 @@ jobs: - name: Check working cluster run: kubectl get pods -A - - name: Install MySQL - working-directory: ./helm/mysql - run: | - helm repo add mysql-operator https://mysql.github.io/mysql-operator/ - helm repo update - helm install mysql-operator mysql-operator/mysql-operator --version "2.1.3" --wait \ - --create-namespace --namespace=mysql-operator - helm install mysql mysql-operator/mysql-innodbcluster --version "2.1.3" --wait \ - --create-namespace --namespace=mysql -f values.yaml - --set-string credentials.root.password=${{ secrets.MYSQL_ROOT_PASSWORD }} + # - name: Install MySQL + # working-directory: ./helm/mysql + # run: | + # helm repo add mysql-operator https://mysql.github.io/mysql-operator/ + # helm repo update + # helm install mysql-operator mysql-operator/mysql-operator --version "2.1.3" --wait \ + # --create-namespace --namespace=mysql-operator + # helm install mysql mysql-operator/mysql-innodbcluster --version "2.1.3" --wait \ + # --create-namespace --namespace=mysql -f values.yaml + # --set-string credentials.root.password=${{ secrets.MYSQL_ROOT_PASSWORD }} - name: Install stunner working-directory: ./scripts/localenv @@ -96,61 +96,61 @@ jobs: working-directory: ./operator run: make deploy IMG=${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.SUB_NAMESPACE }}/operator:${{ env.LABEL }} - - name: Wait for MySQL to be ready - run: | - while true; do - POD_STATUS=$(kubectl get pod mysql-0 -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); - if [ "$POD_STATUS" ]; then - echo "Pod mysql-0 has been created with status: $POD_STATUS"; - break; - else - echo "Waiting for pod mysql-0 to be created..."; - sleep 5; - fi - done - kubectl wait --for=condition=Ready pod/mysql-0 -n mysql --timeout=120s + # - name: Wait for MySQL to be ready + # run: | + # while true; do + # POD_STATUS=$(kubectl get pod mysql-0 -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); + # if [ "$POD_STATUS" ]; then + # echo "Pod mysql-0 has been created with status: $POD_STATUS"; + # break; + # else + # echo "Waiting for pod mysql-0 to be created..."; + # sleep 5; + # fi + # done + # kubectl wait --for=condition=Ready pod/mysql-0 -n mysql --timeout=120s - while true; do - POD_STATUS=$(kubectl get pod -l app.kubernetes.io/component=router -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); \ - if [ "$POD_STATUS" ]; then - echo "MySQL router has been created with status: $POD_STATUS"; - break; - else - echo "Waiting for MySQL router to be created..."; - sleep 5; - fi - done - kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=router -n mysql --timeout=120s - - - name: Install API - working-directory: ./helm/api - run: | - helm install -f values.yaml \ - --set-string env.mysqlRootPassword=${{ secrets.MYSQL_ROOT_PASSWORD }} \ - --set-string env.azureTenantId=${{ secrets.AZURE_TENANT_ID }} - --set-string env.azureClientId=${{ secrets.CLIENT_ID }} - --set-string env.azureClientSecret=${{ secrets.CLIENT_SECRET }} - --set-string env.azureStorageAccount=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }} - --set-string env.azureContainerName=${{ secrets.AZURERM_CONTAINER_NAME }} - --set-string env.azureAksClusterName=${{ secrets.AZURERM_AKS_CLUSTER_NAME }} - --set-string env.azurermSubscriptionId=${{ secrets.AZURERM_SUBSCRIPTION_ID }} - --set-string env.azurermResourceGroupName=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }} - --set-string image.label=${{ env.LABEL }} \ - api . - - - name: Wait for external IP of API - run: | - until [ -n "$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do - sleep 5 - done - - - name: Install frontend - working-directory: ./helm/frontend - run: | - helm install -f values.yaml \ - --set-string appConfig.apiUrl=http://$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$(kubectl get svc api -n api -o jsonpath='{.spec.ports[0].port}') \ - --set-string image.label=${{ env.LABEL }} \ - frontend . + # while true; do + # POD_STATUS=$(kubectl get pod -l app.kubernetes.io/component=router -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); \ + # if [ "$POD_STATUS" ]; then + # echo "MySQL router has been created with status: $POD_STATUS"; + # break; + # else + # echo "Waiting for MySQL router to be created..."; + # sleep 5; + # fi + # done + # kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=router -n mysql --timeout=120s + + # - name: Install API + # working-directory: ./helm/api + # run: | + # helm install -f values.yaml \ + # --set-string env.mysqlRootPassword=${{ secrets.MYSQL_ROOT_PASSWORD }} \ + # --set-string env.azureTenantId=${{ secrets.AZURE_TENANT_ID }} + # --set-string env.azureClientId=${{ secrets.CLIENT_ID }} + # --set-string env.azureClientSecret=${{ secrets.CLIENT_SECRET }} + # --set-string env.azureStorageAccount=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }} + # --set-string env.azureContainerName=${{ secrets.AZURERM_CONTAINER_NAME }} + # --set-string env.azureAksClusterName=${{ secrets.AZURERM_AKS_CLUSTER_NAME }} + # --set-string env.azurermSubscriptionId=${{ secrets.AZURERM_SUBSCRIPTION_ID }} + # --set-string env.azurermResourceGroupName=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }} + # --set-string image.label=${{ env.LABEL }} \ + # api . + + # - name: Wait for external IP of API + # run: | + # until [ -n "$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do + # sleep 5 + # done + + # - name: Install frontend + # working-directory: ./helm/frontend + # run: | + # helm install -f values.yaml \ + # --set-string appConfig.apiUrl=http://$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$(kubectl get svc api -n api -o jsonpath='{.spec.ports[0].port}') \ + # --set-string image.label=${{ env.LABEL }} \ + # frontend . - name: Logout of Azure run: az logout From e51e2ce177925ba68d307d385caa049793d91c87 Mon Sep 17 00:00:00 2001 From: Jonas Konrad <24313231+jalaka@users.noreply.github.com> Date: Sat, 15 Jun 2024 12:25:12 +0200 Subject: [PATCH 38/38] Fix mysql operator newline --- .github/workflows/deploy-az.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 92f5444..fbfff18 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -73,16 +73,16 @@ jobs: - name: Check working cluster run: kubectl get pods -A - # - name: Install MySQL - # working-directory: ./helm/mysql - # run: | - # helm repo add mysql-operator https://mysql.github.io/mysql-operator/ - # helm repo update - # helm install mysql-operator mysql-operator/mysql-operator --version "2.1.3" --wait \ - # --create-namespace --namespace=mysql-operator - # helm install mysql mysql-operator/mysql-innodbcluster --version "2.1.3" --wait \ - # --create-namespace --namespace=mysql -f values.yaml - # --set-string credentials.root.password=${{ secrets.MYSQL_ROOT_PASSWORD }} + - name: Install MySQL + working-directory: ./helm/mysql + run: | + helm repo add mysql-operator https://mysql.github.io/mysql-operator/ + helm repo update + helm install mysql-operator mysql-operator/mysql-operator --version "2.1.3" --wait \ + --create-namespace --namespace=mysql-operator + helm install mysql mysql-operator/mysql-innodbcluster --version "2.1.3" --wait \ + --create-namespace --namespace=mysql -f values.yaml \ + --set-string credentials.root.password=${{ secrets.MYSQL_ROOT_PASSWORD }} - name: Install stunner working-directory: ./scripts/localenv