diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml new file mode 100644 index 0000000..fbfff18 --- /dev/null +++ b/.github/workflows/deploy-az.yml @@ -0,0 +1,193 @@ +name: Deploy to azure + +on: + workflow_dispatch: + inputs: + purge: + description: 'Delete az infrastructure' + required: false + type: boolean + +env: + REGISTRY: ghcr.io + NAMESPACE: austriandatalab + SUB_NAMESPACE: indiegamestream + LABEL: sha-f641ffb9ebff0a3f8c8f9b968bfd50f83a316370 +jobs: + deploy: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.purge == 'false' }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: latest + + - name: Install Helm + uses: azure/setup-helm@v4.2.0 + with: + version: 'latest' + id: install1 + + - name: Install kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'latest' + id: install2 + + + - name: Login to Azure + run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + + - name: Terraform Apply + working-directory: ./iac + run: | + export ARM_CLIENT_SECRET=${{ secrets.CLIENT_SECRET }} + export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} + export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} + export TF_VAR_tenant_id=${{ secrets.AZURERM_TENANT_ID }} + export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " + terraform init + terraform plan -out=tfplan.bin -input=false + terraform apply -auto-approve "tfplan.bin" + - name: Apply tailscale operator + working-directory: ./iac + run: | + az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm uninstall tailscale-operator --namespace=tailscale || true" + az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait" + - name: Connect to tailscale + uses: tailscale/github-action@v2 + with: + oauth-client-id: ${{secrets.TAILSCALE_CLIENT_ID_2}} + oauth-secret: ${{secrets.TAILSCALE_CLIENT_SECRET_2}} + tags: tag:ci + - name: Configure kubernetes config + run: tailscale configure kubeconfig tailscale-operator + + - name: Check working cluster + run: kubectl get pods -A + + - name: Install MySQL + working-directory: ./helm/mysql + run: | + helm repo add mysql-operator https://mysql.github.io/mysql-operator/ + helm repo update + helm install mysql-operator mysql-operator/mysql-operator --version "2.1.3" --wait \ + --create-namespace --namespace=mysql-operator + helm install mysql mysql-operator/mysql-innodbcluster --version "2.1.3" --wait \ + --create-namespace --namespace=mysql -f values.yaml \ + --set-string credentials.root.password=${{ secrets.MYSQL_ROOT_PASSWORD }} + + - name: Install stunner + working-directory: ./scripts/localenv + run: make install_stunner + + - name: Install game operator manifests + working-directory: ./operator + run: make install + + - name: Deploy game operator + working-directory: ./operator + run: make deploy IMG=${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.SUB_NAMESPACE }}/operator:${{ env.LABEL }} + + # - name: Wait for MySQL to be ready + # run: | + # while true; do + # POD_STATUS=$(kubectl get pod mysql-0 -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); + # if [ "$POD_STATUS" ]; then + # echo "Pod mysql-0 has been created with status: $POD_STATUS"; + # break; + # else + # echo "Waiting for pod mysql-0 to be created..."; + # sleep 5; + # fi + # done + # kubectl wait --for=condition=Ready pod/mysql-0 -n mysql --timeout=120s + + # while true; do + # POD_STATUS=$(kubectl get pod -l app.kubernetes.io/component=router -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); \ + # if [ "$POD_STATUS" ]; then + # echo "MySQL router has been created with status: $POD_STATUS"; + # break; + # else + # echo "Waiting for MySQL router to be created..."; + # sleep 5; + # fi + # done + # kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=router -n mysql --timeout=120s + + # - name: Install API + # working-directory: ./helm/api + # run: | + # helm install -f values.yaml \ + # --set-string env.mysqlRootPassword=${{ secrets.MYSQL_ROOT_PASSWORD }} \ + # --set-string env.azureTenantId=${{ secrets.AZURE_TENANT_ID }} + # --set-string env.azureClientId=${{ secrets.CLIENT_ID }} + # --set-string env.azureClientSecret=${{ secrets.CLIENT_SECRET }} + # --set-string env.azureStorageAccount=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }} + # --set-string env.azureContainerName=${{ secrets.AZURERM_CONTAINER_NAME }} + # --set-string env.azureAksClusterName=${{ secrets.AZURERM_AKS_CLUSTER_NAME }} + # --set-string env.azurermSubscriptionId=${{ secrets.AZURERM_SUBSCRIPTION_ID }} + # --set-string env.azurermResourceGroupName=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }} + # --set-string image.label=${{ env.LABEL }} \ + # api . + + # - name: Wait for external IP of API + # run: | + # until [ -n "$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do + # sleep 5 + # done + + # - name: Install frontend + # working-directory: ./helm/frontend + # run: | + # helm install -f values.yaml \ + # --set-string appConfig.apiUrl=http://$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$(kubectl get svc api -n api -o jsonpath='{.spec.ports[0].port}') \ + # --set-string image.label=${{ env.LABEL }} \ + # frontend . + + - name: Logout of Azure + run: az logout + purge: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.purge == 'true' }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: latest + + + - name: Login to Azure + run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + + + + - name: Terraform Apply + working-directory: ./iac + run: | + export ARM_CLIENT_SECRET=${{ secrets.CLIENT_SECRET }} + export ARM_CLIENT_ID=${{ secrets.CLIENT_ID }} + export TF_VAR_subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }} + export TF_VAR_tenant_id=${{ secrets.AZURERM_TENANT_ID }} + export TF_CLI_ARGS_init=" -backend-config=\"resource_group_name=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }}\" -backend-config=\"key=${{ secrets.KEY }}.tfstate\" -backend-config=\"storage_account_name=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }}\" -backend-config=\"container_name=tfbootstrapadmin\" -backend-config=\"subscription_id=${{ secrets.AZURERM_SUBSCRIPTION_ID }}\" -backend-config=\"tenant_id=${{ secrets.AZURERM_TENANT_ID }}\" " + terraform init + terraform plan -out=tfplan.bin -input=false + terraform destroy -auto-approve + terraform destroy -auto-approve + terraform destroy -auto-approve + + - name: Logout of Azure + run: az logout \ No newline at end of file diff --git a/helm/api/templates/api-deployment.yaml b/helm/api/templates/api-deployment.yaml index 2e12126..741c5fd 100644 --- a/helm/api/templates/api-deployment.yaml +++ b/helm/api/templates/api-deployment.yaml @@ -31,8 +31,24 @@ spec: - name: MYSQL_ROOT_USER value: {{ .Values.env.mysqlRootUser | quote }} - name: MYSQL_ROOT_PASSWORD - value: {{ .Values.env.mysqlRootPassword | quote }} + value: {{ required ".Values.env.mysqlRootPassword is required." .Values.env.mysqlRootPassword | quote }} - name: MYSQL_HOST value: {{ .Values.env.mysqlHost | quote }} - name: MYSQL_PORT - value: {{ .Values.env.mysqlPort | quote }} \ No newline at end of file + value: {{ .Values.env.mysqlPort | quote }} + - name: AZURE_TENANT_ID + value: {{ required ".Values.env.azureTenantId is required." .Values.env.azureTenantId | quote }} + - name: AZURE_CLIENT_ID + value: {{ required ".Values.env.azureClientId is required." .Values.env.azureClientId | quote }} + - name: AZURE_CLIENT_SECRET + value: {{ required ".Values.env.azureClientSecret is required." .Values.env.azureClientSecret | quote }} + - name: AZURE_STORAGE_ACCOUNT + value: {{ required ".Values.env.azureStorageAccount is required." .Values.env.azureStorageAccount | quote }} + - name: AZURE_CONTAINER_NAME + value: {{ required ".Values.env.azureContainerName is required." .Values.env.azureContainerName | quote }} + - name: AZURE_AKS_CLUSTER_NAME + value: {{ required ".Values.env.azureAksClusterName is required." .Values.env.azureAksClusterName | quote }} + - name: AZURERM_SUBSCRIPTION_ID + value: {{ required ".Values.env.azurermSubscriptionId is required." .Values.env.azurermSubscriptionId | quote }} + - name: AZURERM_RESOURCE_GROUP_NAME + value: {{ required ".Values.env.azurermResourceGroupName is required." .Values.env.azurermResourceGroupName | quote }} \ No newline at end of file diff --git a/helm/api/templates/api-service.yaml b/helm/api/templates/api-service.yaml index cfd960d..f7ed7b1 100644 --- a/helm/api/templates/api-service.yaml +++ b/helm/api/templates/api-service.yaml @@ -5,8 +5,13 @@ metadata: namespace: {{ .Values.appName }} labels: app: {{ .Values.appName }} + annotations: + tailscale.com/hostname: "api" spec: type: LoadBalancer + {{ if .Values.service.spec.loadBalancerClass }} + loadBalancerClass: {{ .Values.service.spec.loadBalancerClass }} + {{ end }} selector: app: {{ .Values.appName }} ports: diff --git a/helm/api/values-dev.yaml b/helm/api/values-dev.yaml index 955540b..5295723 100644 --- a/helm/api/values-dev.yaml +++ b/helm/api/values-dev.yaml @@ -3,6 +3,10 @@ image: label: localenv pullPolicy: Never +service: + spec: + loadBalancerClass: null + env: ginMode: debug mysqlRootPassword: root \ No newline at end of file diff --git a/helm/api/values.yaml b/helm/api/values.yaml index fe4a44a..69e93a1 100644 --- a/helm/api/values.yaml +++ b/helm/api/values.yaml @@ -7,6 +7,10 @@ image: label: v1.0.0 # Change to proper version tag pullPolicy: IfNotPresent +service: + spec: + loadBalancerClass: tailscale + env: ginMode: release mysqlDatabase: api diff --git a/helm/frontend/templates/frontend-service.yaml b/helm/frontend/templates/frontend-service.yaml index cfd960d..2aa6693 100644 --- a/helm/frontend/templates/frontend-service.yaml +++ b/helm/frontend/templates/frontend-service.yaml @@ -5,8 +5,13 @@ metadata: namespace: {{ .Values.appName }} labels: app: {{ .Values.appName }} + annotations: + tailscale.com/hostname: "frontend" spec: type: LoadBalancer + {{ if .Values.service.spec.loadBalancerClass }} + loadBalancerClass: {{ .Values.service.spec.loadBalancerClass }} + {{ end }} selector: app: {{ .Values.appName }} ports: diff --git a/helm/frontend/values-dev.yaml b/helm/frontend/values-dev.yaml index 0965df8..b62e761 100644 --- a/helm/frontend/values-dev.yaml +++ b/helm/frontend/values-dev.yaml @@ -3,5 +3,9 @@ image: label: localenv pullPolicy: Never +service: + spec: + loadBalancerClass: null + appConfig: production: false \ No newline at end of file diff --git a/helm/frontend/values.yaml b/helm/frontend/values.yaml index a6bc86a..cca23f0 100644 --- a/helm/frontend/values.yaml +++ b/helm/frontend/values.yaml @@ -7,6 +7,10 @@ image: label: v1.0.0 # Change to proper version tag pullPolicy: IfNotPresent +service: + spec: + loadBalancerClass: tailscale + webRootDirectory: /usr/share/nginx/html appConfig: diff --git a/helm/mysql/values.yaml b/helm/mysql/values.yaml index e69de29..d8bf4cc 100644 --- a/helm/mysql/values.yaml +++ b/helm/mysql/values.yaml @@ -0,0 +1,4 @@ +serverInstances: 1 + +tls: + useSelfSigned: true \ No newline at end of file diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf new file mode 100644 index 0000000..8cdb75b --- /dev/null +++ b/iac/kubernetes.tf @@ -0,0 +1,38 @@ +resource "azurerm_kubernetes_cluster" "testCluster" { + name = var.cluster_name + location = data.azurerm_resource_group.rgruntime.location + resource_group_name = data.azurerm_resource_group.rgruntime.name + dns_prefix = var.cluster_name + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_B2ms" + upgrade_settings { + drain_timeout_in_minutes = 0 + max_surge = "10%" + node_soak_duration_in_minutes = 0 + } + } + + network_profile { + network_plugin = "azure" + load_balancer_sku = "standard" + outbound_type = "loadBalancer" + } + + identity { + type = "SystemAssigned" + } + + private_cluster_enabled = true +} + +/* +output "client_certificate" { + value = azurerm_kubernetes_cluster.testCluster.kube_config.0.client_certificate +} + +output "kube_config" { + value = azurerm_kubernetes_cluster.testCluster.kube_config_raw +}*/ \ No newline at end of file diff --git a/iac/main.tf b/iac/main.tf new file mode 100644 index 0000000..f0aa45f --- /dev/null +++ b/iac/main.tf @@ -0,0 +1,45 @@ + +variable "subscription_id" { + type = string +} +variable "tenant_id" { + type = string +} + + + +provider "azurerm" { + subscription_id = var.subscription_id + tenant_id = var.tenant_id + features { + key_vault { + purge_soft_delete_on_destroy = true + recover_soft_deleted_key_vaults = true + } + } +} + +provider "helm" { + kubernetes { + host = azurerm_kubernetes_cluster.testCluster.kube_config.0.host + client_certificate = base64decode(azurerm_kubernetes_cluster.testCluster.kube_config.0.client_certificate) + client_key = base64decode(azurerm_kubernetes_cluster.testCluster.kube_config.0.client_key) + cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.testCluster.kube_config.0.cluster_ca_certificate) + } +} + +terraform { + backend "azurerm" { + use_azuread_auth = true + } +} + + +locals { + common_tags = { + release = "HandsOnCloudNative" + purpose = "class" + classification = "sensitive" + central = "yes" + } +} \ No newline at end of file diff --git a/iac/nsg.tf b/iac/nsg.tf new file mode 100644 index 0000000..41de0e6 --- /dev/null +++ b/iac/nsg.tf @@ -0,0 +1,23 @@ + +resource azurerm_network_security_group "student" { + name = "student-network-security-group" + location = var.globals.location + resource_group_name = data.azurerm_resource_group.rgruntime.name + + #tags = local.common_tags +} + +resource "azurerm_network_security_rule" "lab_nsg" { + name = "Tailscale" + description = "Tailscale UDP port for direct connections. Reduces latency." + priority = 1010 + direction = "Inbound" + access = "Allow" + protocol = "Udp" + source_port_range = "*" + destination_port_range = 41641 + source_address_prefix = "*" + destination_address_prefix = "*" + resource_group_name = data.azurerm_resource_group.rgruntime.name + network_security_group_name = azurerm_network_security_group.student.name +} \ No newline at end of file diff --git a/iac/rg.tf b/iac/rg.tf new file mode 100644 index 0000000..7494082 --- /dev/null +++ b/iac/rg.tf @@ -0,0 +1,82 @@ + + +data "azurerm_client_config" "current" {} + + +#Read my target RG + +data "azurerm_resource_group" "rgruntime" { + name = "rg-service-not2day" + +} + +################################################################ +# IF you are generating any secrets, you need to put them somewhere +# most ideally, you put them into a keyvault of the same lifecycle-stage as the asset the key belongs to +################################################################## + + + +resource "azurerm_key_vault" "kvservice" { + name = "kv-service-not2day-3" + location = data.azurerm_resource_group.rgruntime.location + resource_group_name = data.azurerm_resource_group.rgruntime.name + enabled_for_disk_encryption = true + tenant_id = data.azurerm_client_config.current.tenant_id + soft_delete_retention_days = 7 + purge_protection_enabled = false + + sku_name = "standard" + + + network_acls { + ip_rules = ["0.0.0.0/0" ] #change this + default_action= "Deny" + bypass = "AzureServices" + } + + + access_policy { + tenant_id = data.azurerm_client_config.current.tenant_id + ## Students you must look up your Users Object id in Entra ID and put it here + object_id = var.myuser + + + + secret_permissions = [ + "Get", + "List", + "Restore", + "Delete", + "Set", + "Recover", + "Backup", + ] + + + } + # We are giving this SP access over the current vault + access_policy { + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azurerm_client_config.current.object_id + + + + secret_permissions = [ + "Get", + "List", + "Restore", + "Delete", + "Set", + "Recover", + "Backup", + ] + + + } + + +tags = local.common_tags + +} + diff --git a/iac/variables.tf b/iac/variables.tf new file mode 100644 index 0000000..5fb85ae --- /dev/null +++ b/iac/variables.tf @@ -0,0 +1,16 @@ +variable "globals" { + type = map(any) + + default = { + location = "West Europe" + } +} + +variable "myuser"{ + default = "56ea78b9-6d9f-495b-85ac-7caa86ccc191" +} + +variable "cluster_name" { + type = string + default = "indiegamestream-cluster" +} \ No newline at end of file diff --git a/iac/versions.tf b/iac/versions.tf new file mode 100644 index 0000000..4175fb1 --- /dev/null +++ b/iac/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_providers { + + azuread = { + source = "hashicorp/azuread" + version = ">= 2.7.0" + } + azurerm = { + source = "hashicorp/azurerm" + version = ">= 2.59.0" + } + tls = { + source = "hashicorp/tls" + version = "4.0.4" + } + } + required_version = ">= 0.13" +} diff --git a/operator/config/manager/kustomization.yaml b/operator/config/manager/kustomization.yaml index 5c5f0b8..ad13e96 100644 --- a/operator/config/manager/kustomization.yaml +++ b/operator/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: controller + newTag: latest diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index 3b9b22f..2e3d502 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -4,6 +4,39 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - stream.indiegamestream.com resources: @@ -30,6 +63,18 @@ rules: - get - patch - update +- apiGroups: + - stunner.l7mp.io + resources: + - gatewayconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - stunner.l7mp.io resources: diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index 409d154..ffa97d5 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -47,6 +47,10 @@ type GameReconciler struct { //+kubebuilder:rbac:groups=stream.indiegamestream.com,resources=games/status,verbs=get;update;patch //+kubebuilder:rbac:groups=stream.indiegamestream.com,resources=games/finalizers,verbs=update //+kubebuilder:rbac:groups=stunner.l7mp.io,resources=udproutes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=stunner.l7mp.io,resources=gatewayconfigs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create; // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -497,6 +501,9 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r } func (r *GameReconciler) constructLoadBalancer(game *streamv1.Game, name string, selector string, port int32) (*corev1.Service, error) { + + //className := "tailscale" + svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -504,6 +511,7 @@ func (r *GameReconciler) constructLoadBalancer(game *streamv1.Game, name string, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{"app": selector}, + // LoadBalancerClass: &className, Ports: []corev1.ServicePort{ { Port: port,