diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 146a975..689fcd3 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -8,11 +8,6 @@ on: required: false type: boolean -env: - REGISTRY: ghcr.io - NAMESPACE: austriandatalab - SUB_NAMESPACE: indiegamestream - LABEL: sha-350add069a4899d95e950578b37b43e2fd092fd2 jobs: deploy: runs-on: ubuntu-latest @@ -28,23 +23,15 @@ jobs: with: terraform_version: latest - - name: Install Helm - uses: azure/setup-helm@v4.2.0 - with: - version: 'latest' - id: install1 - - name: Install kubectl uses: azure/setup-kubectl@v3 with: version: 'latest' - id: install2 - + id: install1 - name: Login to Azure run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} - - name: Terraform Apply working-directory: ./iac run: | @@ -56,111 +43,30 @@ jobs: terraform init terraform plan -out=tfplan.bin -input=false terraform apply -auto-approve "tfplan.bin" + - name: Apply tailscale operator working-directory: ./iac run: | - az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait || true" + az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait" + - name: Connect to tailscale uses: tailscale/github-action@v2 with: oauth-client-id: ${{secrets.TAILSCALE_CLIENT_ID_2}} oauth-secret: ${{secrets.TAILSCALE_CLIENT_SECRET_2}} tags: tag:ci + - name: Configure kubernetes config run: tailscale configure kubeconfig tailscale-operator - name: Check working cluster - run: kubectl get pods -A - - - name: Install MySQL - working-directory: ./helm/mysql - run: | - helm repo add mysql-operator https://mysql.github.io/mysql-operator/ - helm repo update - helm install mysql-operator mysql-operator/mysql-operator --version "2.1.3" --wait \ - --create-namespace --namespace=mysql-operator || true - helm install mysql mysql-operator/mysql-innodbcluster --version "2.1.3" --wait \ - --create-namespace --namespace=mysql -f values.yaml \ - --set-string credentials.root.password=${{ secrets.MYSQL_ROOT_PASSWORD }} || true - - - name: Install Open Policy Gatekeeper - working-directory: ./scripts/opa - run: | - helm repo add gatekeeper https://open-policy-agent.github.io/gatekeeper/charts - helm repo update - helm install gatekeeper gatekeeper/gatekeeper --namespace gatekeeper-system --create-namespace --wait || true - kubectl apply -f loadbalancerclass_mutator.yaml - - - name: Install stunner - working-directory: ./scripts/localenv - run: make install_stunner || true - - - name: Install game operator manifests - working-directory: ./operator - run: make install - - - name: Deploy game operator - working-directory: ./operator - run: make deploy IMG=${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.SUB_NAMESPACE }}/operator:${{ env.LABEL }} - - # - name: Wait for MySQL to be ready - # run: | - # while true; do - # POD_STATUS=$(kubectl get pod mysql-0 -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); - # if [ "$POD_STATUS" ]; then - # echo "Pod mysql-0 has been created with status: $POD_STATUS"; - # break; - # else - # echo "Waiting for pod mysql-0 to be created..."; - # sleep 5; - # fi - # done - # kubectl wait --for=condition=Ready pod/mysql-0 -n mysql --timeout=120s - - # while true; do - # POD_STATUS=$(kubectl get pod -l app.kubernetes.io/component=router -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); \ - # if [ "$POD_STATUS" ]; then - # echo "MySQL router has been created with status: $POD_STATUS"; - # break; - # else - # echo "Waiting for MySQL router to be created..."; - # sleep 5; - # fi - # done - # kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=router -n mysql --timeout=120s - - # - name: Install API - # working-directory: ./helm/api - # run: | - # helm install -f values.yaml \ - # --set-string env.mysqlRootPassword=${{ secrets.MYSQL_ROOT_PASSWORD }} \ - # --set-string env.azureTenantId=${{ secrets.AZURE_TENANT_ID }} - # --set-string env.azureClientId=${{ secrets.CLIENT_ID }} - # --set-string env.azureClientSecret=${{ secrets.CLIENT_SECRET }} - # --set-string env.azureStorageAccount=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }} - # --set-string env.azureContainerName=${{ secrets.AZURERM_CONTAINER_NAME }} - # --set-string env.azureAksClusterName=${{ secrets.AZURERM_AKS_CLUSTER_NAME }} - # --set-string env.azurermSubscriptionId=${{ secrets.AZURERM_SUBSCRIPTION_ID }} - # --set-string env.azurermResourceGroupName=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }} - # --set-string image.label=${{ env.LABEL }} \ - # api . - - # - name: Wait for external IP of API - # run: | - # until [ -n "$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do - # sleep 5 - # done - - # - name: Install frontend - # working-directory: ./helm/frontend - # run: | - # helm install -f values.yaml \ - # --set-string appConfig.apiUrl=http://$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$(kubectl get svc api -n api -o jsonpath='{.spec.ports[0].port}') \ - # --set-string image.label=${{ env.LABEL }} \ - # frontend . + run: kubectl get pods -A - name: Logout of Azure run: az logout + + + purge: runs-on: ubuntu-latest if: ${{ github.event.inputs.purge == 'true' }} @@ -175,13 +81,9 @@ jobs: with: terraform_version: latest - - name: Login to Azure run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} - - - - name: Terraform Apply working-directory: ./iac run: | @@ -197,4 +99,5 @@ jobs: terraform destroy -auto-approve || true - name: Logout of Azure + if: always() run: az logout \ No newline at end of file diff --git a/.github/workflows/install-components-az.yml b/.github/workflows/install-components-az.yml new file mode 100644 index 0000000..9c1e847 --- /dev/null +++ b/.github/workflows/install-components-az.yml @@ -0,0 +1,166 @@ +name: Install components to Azure + +on: + workflow_dispatch: + inputs: + component: + type: choice + description: Which component + required: true + options: + - frontend + - api + - api-mysql + - operator + - operator-stunner + - stunner + - mysql + - grafana + - all + label: + type: string + description: Image label to use + +env: + REGISTRY: ghcr.io + NAMESPACE: austriandatalab + SUB_NAMESPACE: indiegamestream +jobs: + install: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Helm + uses: azure/setup-helm@v4.2.0 + with: + version: 'latest' + id: install1 + + - name: Install kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'latest' + id: install2 + + - name: Login to Azure + run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + - name: Connect to tailscale + uses: tailscale/github-action@v2 + with: + oauth-client-id: ${{secrets.TAILSCALE_CLIENT_ID_2}} + oauth-secret: ${{secrets.TAILSCALE_CLIENT_SECRET_2}} + tags: tag:ci + + - name: Configure kubernetes config + run: tailscale configure kubeconfig tailscale-operator + + - name: Install Grafana + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'grafana') }} + run: | + helm repo add grafana https://grafana.github.io/helm-charts + helm repo update + helm install --set-string service.loadBalancerClass=tailscale \ + --set service.type=LoadBalancer \ + --set-string adminPassword=${{ secrets.GRAFANA_ADMIN_PASSWORD }} \ + --set-json 'service.annotations={"tailscale.com/hostname": "grafana"}' \ + grafana grafana/grafana --create-namespace --namespace monitoring + + - name: Install MySQL + working-directory: ./helm/mysql + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'mysql') }} + run: | + helm repo add mysql-operator https://mysql.github.io/mysql-operator/ + helm repo update + helm install mysql-operator mysql-operator/mysql-operator --version "2.1.3" --wait \ + --create-namespace --namespace=mysql-operator + helm install mysql mysql-operator/mysql-innodbcluster --version "2.1.3" --wait \ + --create-namespace --namespace=mysql -f values.yaml \ + --set-string credentials.root.password=${{ secrets.MYSQL_ROOT_PASSWORD }} + + - name: Install STUNner + working-directory: ./helm/stunner + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'stunner') }} + run: | + helm repo add stunner https://l7mp.io/stunner + helm repo update + helm dependency . --skip-refresh + helm install stunner . --create-namespace --namespace=stunner + + - name: Install game operator manifests + working-directory: ./operator + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'operator') }} + run: make install + + - name: Deploy game operator + working-directory: ./operator + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'operator') }} + run: make deploy IMG=${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.SUB_NAMESPACE }}/operator:${{ github.event.inputs.label }} + + - name: Wait for MySQL to be ready + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'api') }} + run: | + while true; do + POD_STATUS=$(kubectl get pod mysql-0 -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); + if [ "$POD_STATUS" ]; then + echo "Pod mysql-0 has been created with status: $POD_STATUS"; + break; + else + echo "Waiting for pod mysql-0 to be created..."; + sleep 5; + fi + done + kubectl wait --for=condition=Ready pod/mysql-0 -n mysql --timeout=120s + + while true; do + POD_STATUS=$(kubectl get pod -l app.kubernetes.io/component=router -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); \ + if [ "$POD_STATUS" ]; then + echo "MySQL router has been created with status: $POD_STATUS"; + break; + else + echo "Waiting for MySQL router to be created..."; + sleep 5; + fi + done + kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=router -n mysql --timeout=120s + + - name: Install API + working-directory: ./helm/api + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'api') }} + run: | + helm install -f values.yaml \ + --set-string env.mysqlRootPassword=${{ secrets.MYSQL_ROOT_PASSWORD }} \ + --set-string env.azureTenantId=${{ secrets.AZURERM_TENANT_ID }} \ + --set-string env.azureClientId=${{ secrets.CLIENT_ID }} \ + --set-string env.azureClientSecret=${{ secrets.CLIENT_SECRET }} \ + --set-string env.azureStorageAccount=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }} \ + --set-string env.azureContainerName=${{ secrets.AZURERM_GAME_CONTAINER_NAME }} \ + --set-string env.azureAksClusterName=${{ secrets.AZURERM_AKS_CLUSTER_NAME }} \ + --set-string env.azurermSubscriptionId=${{ secrets.AZURERM_SUBSCRIPTION_ID }} \ + --set-string env.azurermResourceGroupName=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }} \ + --set-string image.label=${{ github.event.inputs.label }} \ + api . + + - name: Wait for external IP of API + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'frontend') }} + run: | + until [ -n "$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do + sleep 5 + done + + - name: Install frontend + working-directory: ./helm/frontend + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'frontend') }} + run: | + helm install -f values.yaml \ + --set-string appConfig.apiUrl=http://$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$(kubectl get svc api -n api -o jsonpath='{.spec.ports[0].port}') \ + --set-string image.label=${{ github.event.inputs.label }} \ + frontend . + + - name: Logout of Azure + if: always() + run: az logout \ No newline at end of file diff --git a/.github/workflows/uninstall-components-az.yml b/.github/workflows/uninstall-components-az.yml new file mode 100644 index 0000000..c7d54c4 --- /dev/null +++ b/.github/workflows/uninstall-components-az.yml @@ -0,0 +1,93 @@ +name: Uninstall components from Azure + +on: + workflow_dispatch: + inputs: + component: + type: choice + description: Which component + required: true + options: + - frontend + - api + - api-mysql + - operator + - operator-stunner + - stunner + - mysql + - grafana + - all + +jobs: + uninstall: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Helm + uses: azure/setup-helm@v4.2.0 + with: + version: 'latest' + id: install1 + + - name: Install kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'latest' + id: install2 + + - name: Login to Azure + run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + - name: Connect to tailscale + uses: tailscale/github-action@v2 + with: + oauth-client-id: ${{secrets.TAILSCALE_CLIENT_ID_2}} + oauth-secret: ${{secrets.TAILSCALE_CLIENT_SECRET_2}} + tags: tag:ci + + - name: Configure kubernetes config + run: tailscale configure kubeconfig tailscale-operator + + - name: Uninstall MySQL + working-directory: ./helm/mysql + if: ${{ !cancelled() && (github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'mysql')) }} + run: | + helm uninstall mysql -n mysql + helm uninstall mysql-operator -n mysql-operator + + - name: Uninstall STUNner + working-directory: ./helm/stunner + if: ${{ !cancelled() && (github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'stunner')) }} + run: helm uninstall stunner -n stunner + + - name: Uninstall game operator manifests + working-directory: ./operator + if: ${{ !cancelled() && (github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'operator')) }} + run: make uninstall + + - name: Undeploy game operator + working-directory: ./operator + if: ${{ !cancelled() && (github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'operator')) }} + run: make undeploy + + - name: Uninstall API + working-directory: ./helm/api + if: ${{ !cancelled() && (github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'api')) }} + run: helm uninstall api + + - name: Uninstall frontend + working-directory: ./helm/frontend + if: ${{ !cancelled() && (github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'frontend')) }} + run: helm uninstall frontend + + - name: Uninstall Grafana + if: ${{ !cancelled() && (github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'grafana')) }} + run: helm uninstall grafana -n monitoring + + - name: Logout of Azure + if: always() + run: az logout \ No newline at end of file diff --git a/.github/workflows/upgrade-components-az.yml b/.github/workflows/upgrade-components-az.yml new file mode 100644 index 0000000..993a6ca --- /dev/null +++ b/.github/workflows/upgrade-components-az.yml @@ -0,0 +1,127 @@ +# Assumes that the component was already installed to the cluster and we only want to update the image version +# MySQL and STUNner should already be installed when using this workflow +name: Upgrade components on Azure + +on: + workflow_dispatch: + inputs: + component: + type: choice + description: Which component + required: true + options: + - frontend + - api + - operator + - all + label: + type: string + description: Image label to use + +env: + REGISTRY: ghcr.io + NAMESPACE: austriandatalab + SUB_NAMESPACE: indiegamestream +jobs: + upgrade: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Helm + uses: azure/setup-helm@v4.2.0 + with: + version: 'latest' + id: install1 + + - name: Install kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'latest' + id: install2 + + - name: Login to Azure + run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + - name: Connect to tailscale + uses: tailscale/github-action@v2 + with: + oauth-client-id: ${{secrets.TAILSCALE_CLIENT_ID_2}} + oauth-secret: ${{secrets.TAILSCALE_CLIENT_SECRET_2}} + tags: tag:ci + + - name: Configure kubernetes config + run: tailscale configure kubeconfig tailscale-operator + + - name: Install game operator manifests + working-directory: ./operator + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'operator') }} + run: make install + + - name: Deploy game operator + working-directory: ./operator + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'operator') }} + run: make deploy IMG=${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.SUB_NAMESPACE }}/operator:${{ github.event.inputs.label }} + + - name: Undeploy game operator + working-directory: ./operator + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'operator') }} + run: make undeploy + + - name: Wait for MySQL to be ready + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'api') }} + run: | + while true; do + POD_STATUS=$(kubectl get pod mysql-0 -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); + if [ "$POD_STATUS" ]; then + echo "Pod mysql-0 has been created with status: $POD_STATUS"; + break; + else + echo "Waiting for pod mysql-0 to be created..."; + sleep 5; + fi + done + kubectl wait --for=condition=Ready pod/mysql-0 -n mysql --timeout=120s + + while true; do + POD_STATUS=$(kubectl get pod -l app.kubernetes.io/component=router -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); \ + if [ "$POD_STATUS" ]; then + echo "MySQL router has been created with status: $POD_STATUS"; + break; + else + echo "Waiting for MySQL router to be created..."; + sleep 5; + fi + done + kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=router -n mysql --timeout=120s + + - name: Install API + working-directory: ./helm/api + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'api') }} + run: | + helm upgrade -f values.yaml \ + --set-string image.label=${{ github.event.inputs.label }} \ + api . + + - name: Wait for external IP of API + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'frontend') }} + run: | + until [ -n "$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do + sleep 5 + done + + - name: Install frontend + working-directory: ./helm/frontend + if: ${{ github.event.inputs.component == 'all' || contains(github.event.inputs.component, 'frontend') }} + run: | + helm upgrade -f values.yaml \ + --set-string appConfig.apiUrl=http://$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$(kubectl get svc api -n api -o jsonpath='{.spec.ports[0].port}') \ + --set-string image.label=${{ github.event.inputs.label }} \ + frontend . + + - name: Logout of Azure + if: always() + run: az logout \ No newline at end of file diff --git a/api/cmd/main.go b/api/cmd/main.go index 42ab015..e9a9df1 100644 --- a/api/cmd/main.go +++ b/api/cmd/main.go @@ -21,6 +21,7 @@ import ( "net/http" "os" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" ) func setupRouter(db *sql.DB, azClient *azblob.Client) *gin.Engine { @@ -112,19 +113,27 @@ func getKubeConfig() armcontainerservice.ManagedClustersClientListClusterUserCre } func k8sClient() client.Client { - kubeConfig := getKubeConfig() - if len(kubeConfig.Kubeconfigs) == 0 { - log.Fatalf("The kubeconfig request was successful but it's response body is empty") - } - if len(kubeConfig.Kubeconfigs) > 1 { - log.Println("WARNING: Multiple kube-config's have been found. The first one will be used.") - } + //Try to read k8s config directly from environment + restConfig, err := config.GetConfig() + if err != nil || restConfig == nil { + //If it fails, try to get it from azure + kubeConfig := getKubeConfig() + if len(kubeConfig.Kubeconfigs) == 0 { + log.Fatalf("The kubeconfig request was successful but it's response body is empty") + } + if len(kubeConfig.Kubeconfigs) > 1 { + log.Println("WARNING: Multiple kube-config's have been found. The first one will be used.") + } - clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeConfig.Kubeconfigs[0].Value) - if err != nil { - log.Fatalf("failed to load kube config: %v", err) + clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeConfig.Kubeconfigs[0].Value) + if err != nil { + log.Fatalf("failed to create client config: %v", err) + } + restConfig, err = clientConfig.ClientConfig() + if err != nil { + log.Fatalf("failed to load kube config: %v", err) + } } - restConfig, err := clientConfig.ClientConfig() k8sc, err := client.New( restConfig, diff --git a/helm/frontend/templates/frontend-configmap.yaml b/helm/frontend/templates/frontend-configmap.yaml index 56f95b4..0113bde 100644 --- a/helm/frontend/templates/frontend-configmap.yaml +++ b/helm/frontend/templates/frontend-configmap.yaml @@ -1,3 +1,5 @@ +{{- required ".Values.appConfig.production is required!" .Values.appConfig.production -}} +{{- required ".Values.appConfig.apiUrl is required!" .Values.appConfig.apiUrl -}} apiVersion: v1 kind: ConfigMap metadata: