From f21e90f574ba31a190acdc81181c45f9082bb183 Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 15 Jun 2024 20:46:32 +0200 Subject: [PATCH 01/25] migration --- api/migrations/2_game_filename.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 api/migrations/2_game_filename.sql diff --git a/api/migrations/2_game_filename.sql b/api/migrations/2_game_filename.sql new file mode 100644 index 0000000..4df0342 --- /dev/null +++ b/api/migrations/2_game_filename.sql @@ -0,0 +1,2 @@ +ALTER TABLE games ADD FileName varchar(512); +INSERT INTO db_state VALUES (2); \ No newline at end of file From 295ee3662bbd0389ea3f7092b02df2acfb971bf6 Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 15 Jun 2024 20:49:10 +0200 Subject: [PATCH 02/25] updated model + gameRepository --- api/models/game.go | 5 +++-- api/repositories/gameRepository.go | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/api/models/game.go b/api/models/game.go index 5ea2fc1..63a6e19 100644 --- a/api/models/game.go +++ b/api/models/game.go @@ -11,5 +11,6 @@ type Game struct { StorageLocation string `json:"storageLocation"` Status shared.GameStatus `json:"status"` Url string `json:"url"` - Owner string `json:"owner"` -} + Owner string `json:"owner"` + FileName string `json:"fileName"` +} diff --git a/api/repositories/gameRepository.go b/api/repositories/gameRepository.go index a2b6b24..34cb212 100644 --- a/api/repositories/gameRepository.go +++ b/api/repositories/gameRepository.go @@ -52,7 +52,7 @@ func (g gameRepository) FindAllByOwner(owner string) ([]models.Game, error) { func (g gameRepository) FindByID(id uuid.UUID) (*models.Game, error) { var game models.Game err := g.db.QueryRow("SELECT * FROM games WHERE ID = ?", id). - Scan(&game.ID, &game.Title, &game.StorageLocation, &game.Status, &game.Url, &game.Owner) + Scan(&game.ID, &game.Title, &game.StorageLocation, &game.Status, &game.Url, &game.Owner, &game.FileName) if err != nil { if err == sql.ErrNoRows { return nil, nil @@ -74,24 +74,24 @@ func (g gameRepository) Save(game *models.Game) error { if existing != nil { //If yes, update the existing entry - stmt, err := g.db.Prepare("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=? WHERE ID = ?") + stmt, err := g.db.Prepare("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=?, FileName=? WHERE ID = ?") if err != nil { return err } - return checkResult(stmt.Exec(game.Title, game.StorageLocation, game.Status, game.Url, game.ID)) + return checkResult(stmt.Exec(game.Title, game.StorageLocation, game.Status, game.Url, game.ID, game.FileName)) } } else { game.ID = uuid.New() } //If not create a new one - stmt, err := g.db.Prepare("INSERT INTO games (ID, Title, StorageLocation, Status, Url, Owner) VALUES (?,?,?,?,?,?)") + stmt, err := g.db.Prepare("INSERT INTO games (ID, Title, StorageLocation, Status, Url, Owner, FileName) VALUES (?,?,?,?,?,?,?)") if err != nil { return err } - return checkResult(stmt.Exec(game.ID, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner)) + return checkResult(stmt.Exec(game.ID, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner, game.FileName)) } // Delete removes the entry with a specific id from the games database. @@ -128,7 +128,7 @@ func readGamesFromRows(query *sql.Rows) ([]models.Game, error) { var games = []models.Game{} for query.Next() { var game models.Game - err := query.Scan(&game.ID, &game.Title, &game.StorageLocation, &game.Status, &game.Url, &game.Owner) + err := query.Scan(&game.ID, &game.Title, &game.StorageLocation, &game.Status, &game.Url, &game.Owner, &game.FileName) if err != nil { return nil, err } From 03d53270b161469baca228ef9e536d7c615775e1 Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 15 Jun 2024 20:55:56 +0200 Subject: [PATCH 03/25] updated repository tests --- api/tests/gameRepository_test.go | 40 +++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/api/tests/gameRepository_test.go b/api/tests/gameRepository_test.go index 42e1d16..e4719d9 100644 --- a/api/tests/gameRepository_test.go +++ b/api/tests/gameRepository_test.go @@ -88,10 +88,11 @@ func Test_Create_Game_Without_Id_Should_Succeed_And_SetId(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectPrepare(regexp.QuoteMeta("INSERT INTO games")) mock.ExpectExec(regexp.QuoteMeta("INSERT INTO games")). - WithArgs(sqlmock.AnyArg(), game.Title, game.StorageLocation, game.Status, game.Url, game.Owner). + WithArgs(sqlmock.AnyArg(), game.Title, game.StorageLocation, game.Status, game.Url, game.Owner, game.FileName). WillReturnResult(sqlmock.NewResult(0, 1)) //Run the test @@ -128,6 +129,7 @@ func Test_Create_Game_With_Id_Should_Succeed(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM games WHERE ID = ?")). @@ -136,7 +138,7 @@ func Test_Create_Game_With_Id_Should_Succeed(t *testing.T) { mock.ExpectPrepare("INSERT INTO games") mock.ExpectExec("INSERT INTO games"). - WithArgs(game.ID, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner). + WithArgs(game.ID, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner, game.FileName). WillReturnResult(sqlmock.NewResult(0, 1)) //Run the test @@ -176,20 +178,21 @@ func Test_Save_Existing_Game_Should_Succeed(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM games WHERE ID = ?")). WithArgs(id).WillReturnRows( - sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner"}). - AddRow(id, "", "", "", "", ""), + sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner", "FileName"}). + AddRow(id, "", "", "", "", "", ""), ) mock.ExpectPrepare(regexp. - QuoteMeta("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=? WHERE ID = ?")) + QuoteMeta("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=?, FileName=? WHERE ID = ?")) mock.ExpectExec(regexp. - QuoteMeta("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=? WHERE ID = ?")). - WithArgs(game.Title, game.StorageLocation, game.Status, game.Url, game.ID). + QuoteMeta("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=?, FileName=? WHERE ID = ?")). + WithArgs(game.Title, game.StorageLocation, game.Status, game.Url, game.ID, game.FileName). WillReturnResult(sqlmock.NewResult(0, 1)) //Run the test @@ -229,12 +232,13 @@ func Test_Find_Game_By_Id_Should_Succeed(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM games WHERE ID = ?")). WithArgs(id).WillReturnRows( - sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner"}). - AddRow(id, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner), + sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner", "FileName"}). + AddRow(id, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner, game.FileName), ) //Run the test @@ -295,6 +299,7 @@ func Test_Find_Two_Games_Of_Same_Owner_Should_Succeed(t *testing.T) { Status: "AMockC", Url: "AMockD", Owner: "MockOwner", + FileName: "TestFile.nes", } gameB := models.Game{ @@ -304,15 +309,16 @@ func Test_Find_Two_Games_Of_Same_Owner_Should_Succeed(t *testing.T) { Status: "BMockC", Url: "BMockD", Owner: "MockOwner", + FileName: "TestFile2.nes", } mock.ExpectPrepare(regexp.QuoteMeta("SELECT * FROM games WHERE owner = ?")) mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM games WHERE owner = ?")). WithArgs("MockOwner"). WillReturnRows( - sqlmock.NewRows([]string{"ID", "Title", "StorageLocation", "Status", "Url", "Owner"}). - AddRow(gameA.ID, gameA.Title, gameA.StorageLocation, gameA.Status, gameA.Url, gameA.Owner). - AddRow(gameB.ID, gameB.Title, gameB.StorageLocation, gameB.Status, gameB.Url, gameB.Owner), + sqlmock.NewRows([]string{"ID", "Title", "StorageLocation", "Status", "Url", "Owner", "FileName"}). + AddRow(gameA.ID, gameA.Title, gameA.StorageLocation, gameA.Status, gameA.Url, gameA.Owner, gameA.FileName). + AddRow(gameB.ID, gameB.Title, gameB.StorageLocation, gameB.Status, gameB.Url, gameB.Owner, gameB.FileName), ) //Run the test @@ -350,7 +356,7 @@ func Test_Find_Games_When_Database_Is_Empty_Should_Succeed(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM games WHERE owner = ?")). WithArgs("MockOwner"). WillReturnRows( - sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner"}), + sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner", "FileName"}), ) //Run the test @@ -389,11 +395,12 @@ func Test_Read_Owner_Should_Succeed(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectQuery(regexp.QuoteMeta("SELECT Owner FROM games WHERE ID = ?")). WithArgs(id). - WillReturnRows(sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner"})) + WillReturnRows(sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner", "FileName"})) //Run the test repository := repositories.GameRepository(db) @@ -427,6 +434,7 @@ func Test_Read_Owner_Should_Throw_When_Database_Is_Empty(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectQuery(regexp.QuoteMeta("SELECT Owner FROM games WHERE ID = ?")). @@ -483,4 +491,8 @@ func compareGames(t *testing.T, game *models.Game, res *models.Game) { t.Errorf("game owner has been changed") } + if game.FileName != res.FileName { + t.Errorf("game file name has been changed") + } + } From 92126735834883b68557090fc60c9abbc36704e2 Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 15 Jun 2024 20:57:55 +0200 Subject: [PATCH 04/25] set filename when creating a game --- api/services/gameService.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/services/gameService.go b/api/services/gameService.go index 369c486..9dcb61e 100644 --- a/api/services/gameService.go +++ b/api/services/gameService.go @@ -53,6 +53,7 @@ func (g gameService) Save(fileHeader *multipart.FileHeader, title string, owner Status: shared.Status_New, Url: "", Owner: owner, + FileName: fileHeader.Filename, } //Upload game to azure blob storage container From d8e6b1e409f93a5f7aa1eb77ef9ca0930e6127c8 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Sat, 15 Jun 2024 15:32:26 +0200 Subject: [PATCH 05/25] Fix loadbalancer IP fetching in operator --- operator/internal/controller/stream/game_controller.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index 316177b..2704f84 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -565,7 +565,10 @@ func waitForLoadBalancerIP(ctx context.Context, k8sClient client.Client, namespa if len(svc.Status.LoadBalancer.Ingress) > 0 { ip = svc.Status.LoadBalancer.Ingress[0].IP if ip != "" { - return true, nil + ip = svc.Status.LoadBalancer.Ingress[1].IP + if ip != "" { + return true, nil + } } if len(svc.Status.LoadBalancer.Ingress) > 1 { ip = svc.Status.LoadBalancer.Ingress[1].IP From fa8093410fb9a4fee8cf8fb25cdde1acc6b45145 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Sat, 15 Jun 2024 15:45:16 +0200 Subject: [PATCH 06/25] Fix panic --- operator/internal/controller/stream/game_controller.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index 2704f84..330016e 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -565,6 +565,9 @@ func waitForLoadBalancerIP(ctx context.Context, k8sClient client.Client, namespa if len(svc.Status.LoadBalancer.Ingress) > 0 { ip = svc.Status.LoadBalancer.Ingress[0].IP if ip != "" { + return true, nil + } + if len(svc.Status.LoadBalancer.Ingress) > 1 { ip = svc.Status.LoadBalancer.Ingress[1].IP if ip != "" { return true, nil From da6b7b1fd8a81d1e96eb3e93d5bf55c78be2484a Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Sat, 15 Jun 2024 16:47:41 +0200 Subject: [PATCH 07/25] Add additional node --- iac/kubernetes.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index 2e2dc41..e5945ff 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -14,7 +14,7 @@ resource "azurerm_kubernetes_cluster" "testCluster" { node_soak_duration_in_minutes = 0 } max_pods = 110 - temporary_name_for_rotation = "upgrade" + temporary_name_for_rotation = "upgrade2" } network_profile { From 42c903222ea9ae8c6e06804211757e8cf230a3de Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Sat, 15 Jun 2024 17:11:07 +0200 Subject: [PATCH 08/25] Change upgrade parameters --- iac/kubernetes.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index e5945ff..2e2dc41 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -14,7 +14,7 @@ resource "azurerm_kubernetes_cluster" "testCluster" { node_soak_duration_in_minutes = 0 } max_pods = 110 - temporary_name_for_rotation = "upgrade2" + temporary_name_for_rotation = "upgrade" } network_profile { From 8d66a76eac4d8df320ec8fb26068e99f37fed70e Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Sat, 15 Jun 2024 18:43:17 +0200 Subject: [PATCH 09/25] Add LoadBalancer Mutator to keep stunner ips in tailscale --- .github/workflows/deploy-az.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 689fcd3..fdcdae0 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -47,8 +47,7 @@ jobs: - name: Apply tailscale operator working-directory: ./iac run: | - az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait" - + az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait || true" - name: Connect to tailscale uses: tailscale/github-action@v2 with: From 5316121caf8043a660b1f26139505cd38760bb85 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Sat, 15 Jun 2024 20:25:24 +0200 Subject: [PATCH 10/25] Remove namespace of tailscale operator --- .github/workflows/deploy-az.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index fdcdae0..6baff66 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -47,7 +47,7 @@ jobs: - name: Apply tailscale operator working-directory: ./iac run: | - az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait || true" + az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait || true" - name: Connect to tailscale uses: tailscale/github-action@v2 with: From 5f814a26cd673077a00f08b9865a60054dcdcc1f Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 17 Jun 2024 01:06:14 +0200 Subject: [PATCH 11/25] Add storage class and kubernetes permissions --- .github/workflows/deploy-az.yml | 102 ++++++++++++++++++ operator/config/rbac/kustomization.yaml | 2 + scripts/azure-csi/initialize-csi-storage.yaml | 37 +++++++ .../api_cluster_permission.yaml | 12 +++ 4 files changed, 153 insertions(+) create mode 100644 scripts/azure-csi/initialize-csi-storage.yaml create mode 100644 scripts/cluster-permissions/api_cluster_permission.yaml diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 6baff66..548a406 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -61,6 +61,105 @@ jobs: - name: Check working cluster run: kubectl get pods -A + - name: Initialize CSI + working-directory: ./scripts/azure-csi + run: | + STORAGE_KEY=$(az storage account keys list --resource-group rg-management-not2day --account-name ${{secrets.AZURERM_GAME_STORAGE_ACCOUNT_NAME}} --query "[0].value" -o tsv) + kubectl create secret generic azure-secret --from-literal=azurestorageaccountname=${{secrets.AZURERM_GAME_STORAGE_ACCOUNT_NAME}} --from-literal=azurestorageaccountkey=$STORAGE_KEY + kubectl apply -f initialize-csi-storage.yaml + + - name: Initialize Cluster permissions + working-directory: ./scripts/cluster-permissions + run: | + kubectl apply -f api_cluster_permission.yaml + + - name: Install MySQL + working-directory: ./helm/mysql + run: | + helm repo add mysql-operator https://mysql.github.io/mysql-operator/ + helm repo update + helm install mysql-operator mysql-operator/mysql-operator --version "2.1.3" --wait \ + --create-namespace --namespace=mysql-operator || true + helm install mysql mysql-operator/mysql-innodbcluster --version "2.1.3" --wait \ + --create-namespace --namespace=mysql -f values.yaml \ + --set-string credentials.root.password=${{ secrets.MYSQL_ROOT_PASSWORD }} || true + + - name: Install Open Policy Gatekeeper + working-directory: ./scripts/opa + run: | + helm repo add gatekeeper https://open-policy-agent.github.io/gatekeeper/charts + helm repo update + helm install gatekeeper gatekeeper/gatekeeper --namespace gatekeeper-system --create-namespace --wait || true + kubectl apply -f loadbalancerclass_mutator.yaml + + - name: Install stunner + working-directory: ./scripts/localenv + run: make install_stunner || true + + - name: Install game operator manifests + working-directory: ./operator + run: make install + + - name: Deploy game operator + working-directory: ./operator + run: make deploy IMG=${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.SUB_NAMESPACE }}/operator:${{ env.LABEL }} + + # - name: Wait for MySQL to be ready + # run: | + # while true; do + # POD_STATUS=$(kubectl get pod mysql-0 -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); + # if [ "$POD_STATUS" ]; then + # echo "Pod mysql-0 has been created with status: $POD_STATUS"; + # break; + # else + # echo "Waiting for pod mysql-0 to be created..."; + # sleep 5; + # fi + # done + # kubectl wait --for=condition=Ready pod/mysql-0 -n mysql --timeout=120s + + # while true; do + # POD_STATUS=$(kubectl get pod -l app.kubernetes.io/component=router -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); \ + # if [ "$POD_STATUS" ]; then + # echo "MySQL router has been created with status: $POD_STATUS"; + # break; + # else + # echo "Waiting for MySQL router to be created..."; + # sleep 5; + # fi + # done + # kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=router -n mysql --timeout=120s + + # - name: Install API + # working-directory: ./helm/api + # run: | + # helm install -f values.yaml \ + # --set-string env.mysqlRootPassword=${{ secrets.MYSQL_ROOT_PASSWORD }} \ + # --set-string env.azureTenantId=${{ secrets.AZURE_TENANT_ID }} + # --set-string env.azureClientId=${{ secrets.CLIENT_ID }} + # --set-string env.azureClientSecret=${{ secrets.CLIENT_SECRET }} + # --set-string env.azureStorageAccount=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }} + # --set-string env.azureContainerName=${{ secrets.AZURERM_CONTAINER_NAME }} + # --set-string env.azureAksClusterName=${{ secrets.AZURERM_AKS_CLUSTER_NAME }} + # --set-string env.azurermSubscriptionId=${{ secrets.AZURERM_SUBSCRIPTION_ID }} + # --set-string env.azurermResourceGroupName=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }} + # --set-string image.label=${{ env.LABEL }} \ + # api . + + # - name: Wait for external IP of API + # run: | + # until [ -n "$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do + # sleep 5 + # done + + # - name: Install frontend + # working-directory: ./helm/frontend + # run: | + # helm install -f values.yaml \ + # --set-string appConfig.apiUrl=http://$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$(kubectl get svc api -n api -o jsonpath='{.spec.ports[0].port}') \ + # --set-string image.label=${{ env.LABEL }} \ + # frontend . + - name: Logout of Azure run: az logout @@ -83,6 +182,9 @@ jobs: - name: Login to Azure run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + + - name: Terraform Apply working-directory: ./iac run: | diff --git a/operator/config/rbac/kustomization.yaml b/operator/config/rbac/kustomization.yaml index 731832a..fc4f042 100644 --- a/operator/config/rbac/kustomization.yaml +++ b/operator/config/rbac/kustomization.yaml @@ -9,6 +9,8 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +- game_editor_role.yaml +- game_viewer_role.yaml # Comment the following 4 lines if you want to disable # the auth proxy (https://github.com/brancz/kube-rbac-proxy) # which protects your /metrics endpoint. diff --git a/scripts/azure-csi/initialize-csi-storage.yaml b/scripts/azure-csi/initialize-csi-storage.yaml new file mode 100644 index 0000000..80963dc --- /dev/null +++ b/scripts/azure-csi/initialize-csi-storage.yaml @@ -0,0 +1,37 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azureblob-sc +provisioner: blob.csi.azure.com +parameters: + containerName: games + storageAccount: indiegamestream + csi.storage.k8s.io/provisioner-secret-name: azure-secret + csi.storage.k8s.io/provisioner-secret-namespace: default + csi.storage.k8s.io/node-stage-secret-name: azure-secret + csi.storage.k8s.io/node-stage-secret-namespace: default +reclaimPolicy: Retain +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: azure-blob-pvc +spec: + accessModes: + - ReadOnlyMany + storageClassName: azureblob-sc + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/scripts/cluster-permissions/api_cluster_permission.yaml b/scripts/cluster-permissions/api_cluster_permission.yaml new file mode 100644 index 0000000..7c6c4d2 --- /dev/null +++ b/scripts/cluster-permissions/api_cluster_permission.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: game-editor-binding +subjects: +- kind: ServiceAccount + name: default + namespace: api +roleRef: + kind: ClusterRole + name: game-editor-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file From 4484cd2c9793c1999f8b16337b33a30fbb6478f0 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Sat, 15 Jun 2024 22:30:46 +0200 Subject: [PATCH 12/25] UserAssigned identities for cluster --- iac/kubernetes.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index 2e2dc41..baabc3f 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -28,7 +28,8 @@ resource "azurerm_kubernetes_cluster" "testCluster" { } identity { - type = "SystemAssigned" + type = "UserAssigned" + identity_ids = [var.myuser, "7ab666bb-6355-4240-aa93-16bfbb9fd5f7"] } private_cluster_enabled = true From cfa49f870a8ed27d7678befbd2c8e3850136bb86 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Sat, 15 Jun 2024 23:11:42 +0200 Subject: [PATCH 13/25] Added Admins --- iac/kubernetes.tf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index baabc3f..90a45fa 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -28,8 +28,13 @@ resource "azurerm_kubernetes_cluster" "testCluster" { } identity { - type = "UserAssigned" - identity_ids = [var.myuser, "7ab666bb-6355-4240-aa93-16bfbb9fd5f7"] + type = "SystemAssigned" + } + + azure_active_directory_role_based_access_control { + managed = true + azure_rbac_enabled = false + admin_group_object_ids = [var.myuser, "7ab666bb-6355-4240-aa93-16bfbb9fd5f7"] } private_cluster_enabled = true From 79a141515ea7674220eb4abe022d99ee2cc7b056 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Sat, 15 Jun 2024 23:19:30 +0200 Subject: [PATCH 14/25] Enable Azure RBAC --- iac/kubernetes.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index 90a45fa..2714f8b 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -33,7 +33,7 @@ resource "azurerm_kubernetes_cluster" "testCluster" { azure_active_directory_role_based_access_control { managed = true - azure_rbac_enabled = false + azure_rbac_enabled = true admin_group_object_ids = [var.myuser, "7ab666bb-6355-4240-aa93-16bfbb9fd5f7"] } From a65b1c152fa7ff882ad5660e94e038d23aa975e7 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Sat, 15 Jun 2024 23:29:15 +0200 Subject: [PATCH 15/25] Enable role based access control --- iac/kubernetes.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index 2714f8b..5a45781 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -37,6 +37,8 @@ resource "azurerm_kubernetes_cluster" "testCluster" { admin_group_object_ids = [var.myuser, "7ab666bb-6355-4240-aa93-16bfbb9fd5f7"] } + role_based_access_control_enabled = true + private_cluster_enabled = true } From 434b180b136a9a5f8945dba55409cdd23a4e30dc Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Sat, 15 Jun 2024 23:56:31 +0200 Subject: [PATCH 16/25] Added admin role assignment --- iac/kubernetes.tf | 11 ++++++++--- iac/variables.tf | 6 ++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index 5a45781..cddb553 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -34,14 +34,19 @@ resource "azurerm_kubernetes_cluster" "testCluster" { azure_active_directory_role_based_access_control { managed = true azure_rbac_enabled = true - admin_group_object_ids = [var.myuser, "7ab666bb-6355-4240-aa93-16bfbb9fd5f7"] + admin_group_object_ids = var.aks_admin_group_object_ids } - role_based_access_control_enabled = true - private_cluster_enabled = true } +resource "azurerm_role_assignment" "admin" { + for_each = toset(var.aks_admin_group_object_ids) + scope = azurerm_kubernetes_cluster.testCluster.id + role_definition_name = "Azure Kubernetes Service Cluster User Role" + principal_id = each.value +} + /* output "client_certificate" { value = azurerm_kubernetes_cluster.testCluster.kube_config.0.client_certificate diff --git a/iac/variables.tf b/iac/variables.tf index 5fb85ae..cf91f16 100644 --- a/iac/variables.tf +++ b/iac/variables.tf @@ -13,4 +13,10 @@ variable "myuser"{ variable "cluster_name" { type = string default = "indiegamestream-cluster" +} + +variable "aks_admin_group_object_ids" { + description = "aks admin group ids" + type = list(string) + default = [var.myuser, "7ab666bb-6355-4240-aa93-16bfbb9fd5f7"] } \ No newline at end of file From d63084126db7511f9e61f11390f0b9d64a0eaad9 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Sat, 15 Jun 2024 23:57:57 +0200 Subject: [PATCH 17/25] Removed variable --- iac/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iac/variables.tf b/iac/variables.tf index cf91f16..929c232 100644 --- a/iac/variables.tf +++ b/iac/variables.tf @@ -18,5 +18,5 @@ variable "cluster_name" { variable "aks_admin_group_object_ids" { description = "aks admin group ids" type = list(string) - default = [var.myuser, "7ab666bb-6355-4240-aa93-16bfbb9fd5f7"] + default = ["56ea78b9-6d9f-495b-85ac-7caa86ccc191", "7ab666bb-6355-4240-aa93-16bfbb9fd5f7"] } \ No newline at end of file From 6135aadb75377a8bcd77d59d3803753429e1ad0d Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Sun, 16 Jun 2024 00:06:32 +0200 Subject: [PATCH 18/25] Removed role assignment --- iac/kubernetes.tf | 7 ------- 1 file changed, 7 deletions(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index cddb553..948a6dc 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -40,13 +40,6 @@ resource "azurerm_kubernetes_cluster" "testCluster" { private_cluster_enabled = true } -resource "azurerm_role_assignment" "admin" { - for_each = toset(var.aks_admin_group_object_ids) - scope = azurerm_kubernetes_cluster.testCluster.id - role_definition_name = "Azure Kubernetes Service Cluster User Role" - principal_id = each.value -} - /* output "client_certificate" { value = azurerm_kubernetes_cluster.testCluster.kube_config.0.client_certificate From f9d7dab4f2ed2139462e56d0557a17342c5de543 Mon Sep 17 00:00:00 2001 From: Thomas Riegler Date: Sun, 16 Jun 2024 14:10:20 +0200 Subject: [PATCH 19/25] Reversed IAC changes of yesterday --- iac/kubernetes.tf | 6 ------ iac/variables.tf | 6 ------ 2 files changed, 12 deletions(-) diff --git a/iac/kubernetes.tf b/iac/kubernetes.tf index 948a6dc..2e2dc41 100644 --- a/iac/kubernetes.tf +++ b/iac/kubernetes.tf @@ -31,12 +31,6 @@ resource "azurerm_kubernetes_cluster" "testCluster" { type = "SystemAssigned" } - azure_active_directory_role_based_access_control { - managed = true - azure_rbac_enabled = true - admin_group_object_ids = var.aks_admin_group_object_ids - } - private_cluster_enabled = true } diff --git a/iac/variables.tf b/iac/variables.tf index 929c232..5fb85ae 100644 --- a/iac/variables.tf +++ b/iac/variables.tf @@ -13,10 +13,4 @@ variable "myuser"{ variable "cluster_name" { type = string default = "indiegamestream-cluster" -} - -variable "aks_admin_group_object_ids" { - description = "aks admin group ids" - type = list(string) - default = ["56ea78b9-6d9f-495b-85ac-7caa86ccc191", "7ab666bb-6355-4240-aa93-16bfbb9fd5f7"] } \ No newline at end of file From f066448d8461b33ec8cf3fe48525e75584b27ac4 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 17 Jun 2024 02:07:03 +0200 Subject: [PATCH 20/25] Change frontend url --- .../controller/stream/game_controller.go | 61 ++++++++++++++++--- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index 330016e..abe6108 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -196,7 +196,13 @@ func (r *GameReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. // Note that Status is a subresource, so changes to it are ignored by the cache, hence the need to update it manually //game.Status.Nodes = nodes //game.Status.Phase = phase - game.Status.URL = fmt.Sprintf("http://%s:8000", coordIP) + outsidehostname, err := waitForLoadBalancerHostname(ctx, r.Client, game.Namespace, coordinatorName) + if err != nil { + log.Error(err, "unable to get LoadBalancer Hostname for Coordinator") + return ctrl.Result{}, err + } + + game.Status.URL = fmt.Sprintf("http://%s", outsidehostname) //TODO: Add nginx ingress url to status if err := r.Status().Update(ctx, game); err != nil { log.Error(err, "unable to update Game status") @@ -479,7 +485,7 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r }, { Name: "CLOUD_GAME_WORKER_NETWORK_COORDINATORADDRESS", - Value: fmt.Sprintf("%s:8000", coordIP), + Value: fmt.Sprintf("%s:80", coordIP), }, { Name: "CLOUD_GAME_WORKER_NETWORK_PUBLICADDRESS", @@ -503,18 +509,26 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r func (r *GameReconciler) constructLoadBalancer(game *streamv1.Game, name string, selector string, port int32) (*corev1.Service, error) { className := "tailscale" + annotation := fmt.Sprintf("%s-%s", game.Spec.Name, game.Name) + annotations := map[string]string{} + outsidePort := port + if selector == "coordinator" { + annotations["tailscale.com/hostname"] = annotation + outsidePort = 80 + } svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: game.Namespace, + Name: name, + Namespace: game.Namespace, + Annotations: annotations, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{"app": selector}, LoadBalancerClass: &className, Ports: []corev1.ServicePort{ { - Port: port, + Port: outsidePort, TargetPort: intstr.FromInt32(port), }, }, @@ -573,9 +587,32 @@ func waitForLoadBalancerIP(ctx context.Context, k8sClient client.Client, namespa return true, nil } } + } + return false, nil + }) + + if err != nil { + return "", err + } + return ip, nil +} + +func waitForLoadBalancerHostname(ctx context.Context, k8sClient client.Client, namespace, serviceName string) (string, error) { + var hostname string + + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 20*time.Second, true, func(ctx context.Context) (bool, error) { + svc := &corev1.Service{} + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: serviceName}, svc); err != nil { + return false, err + } + if len(svc.Status.LoadBalancer.Ingress) > 0 { + hostname = svc.Status.LoadBalancer.Ingress[0].Hostname + if hostname != "" { + return true, nil + } if len(svc.Status.LoadBalancer.Ingress) > 1 { - ip = svc.Status.LoadBalancer.Ingress[1].IP - if ip != "" { + hostname = svc.Status.LoadBalancer.Ingress[1].Hostname + if hostname != "" { return true, nil } } @@ -583,10 +620,16 @@ func waitForLoadBalancerIP(ctx context.Context, k8sClient client.Client, namespa return false, nil }) + //fallback on ip if err != nil { - return "", err + ip, err := waitForLoadBalancerIP(ctx, k8sClient, namespace, serviceName) + if err != nil { + return "", err + } + return ip, nil } - return ip, nil + + return hostname, nil } var ( From 6bc58b32e82e2961aadd3fc4daab5f6ce0717392 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Mon, 17 Jun 2024 12:57:25 +0200 Subject: [PATCH 21/25] Add volume mount to controller --- .../controller/stream/game_controller.go | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index abe6108..2f8a98f 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -403,9 +403,9 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam Containers: []corev1.Container{ { Name: "coordinator", - Image: "valniae/snekyrepo:crdi", - Command: []string{"coordinator"}, - Args: []string{"--v=5"}, + Image: "ghcr.io/giongto35/cloud-game/cloud-game:master", + Command: []string{"./coordinator"}, + Args: []string{""}, Ports: []corev1.ContainerPort{ { ContainerPort: 8000, @@ -417,7 +417,7 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam Value: gatewayConfig.Spec.Password, }, { - Name: "CLOUD_GAME_WEBRTC_ICESERVERS_0_URL", + Name: "CLOUD_GAME_WEBRTC_ICESERVERS_0_URLS", Value: fmt.Sprintf("turn:%s:3478", gatewayIP), }, { @@ -429,7 +429,7 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam Value: gatewayConfig.Spec.Password, }, { - Name: "CLOUD_GAME_WEBRTC_ICESERVERS_1_URL", + Name: "CLOUD_GAME_WEBRTC_ICESERVERS_1_URLS", Value: fmt.Sprintf("turn:%s:3478", gatewayIP), }, { @@ -452,6 +452,8 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam } func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, resourceName string, coordIP string, workerIP string) (*appsv1.Deployment, error) { + fullpath := fmt.Sprintf("/usr/local/share/cloud-game/assets/games/%s", "test.gba") + dep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -470,9 +472,9 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r Containers: []corev1.Container{ { Name: "worker", - Image: "valniae/snekyrepo:crdi", - Command: []string{"worker"}, - Args: []string{"--v=5"}, + Image: "ghcr.io/giongto35/cloud-game/cloud-game:master", + Command: []string{"./worker"}, + Args: []string{""}, Ports: []corev1.ContainerPort{ { ContainerPort: 8443, @@ -491,7 +493,24 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r Name: "CLOUD_GAME_WORKER_NETWORK_PUBLICADDRESS", Value: workerIP, }, - }, //TODO mount game executable and config game config file + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "gamestorage", + MountPath: fullpath, + SubPath: "varooom-3d (1).gba", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "gamestorage", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "azure-blob-pvc", + }, + }, }, }, }, From d36893120d4353b129ab76de0bb9f38c689bce6e Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Tue, 18 Jun 2024 13:44:25 +0200 Subject: [PATCH 22/25] Fix game discovery --- .../controller/stream/game_controller.go | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index 2f8a98f..aec32fb 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -385,6 +385,8 @@ func int32Ptr(i int32) *int32 { } func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Game, resourceName string, gatewayConfig *stunnerv1.GatewayConfig, gatewayIP string) (*appsv1.Deployment, error) { + fullpath := fmt.Sprintf("/usr/local/share/cloud-game/assets/games/%s", "test.gba") + dep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -403,7 +405,7 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam Containers: []corev1.Container{ { Name: "coordinator", - Image: "ghcr.io/giongto35/cloud-game/cloud-game:master", + Image: "ghcr.io/giongto35/cloud-game/cloud-game:v3.0.5", Command: []string{"./coordinator"}, Args: []string{""}, Ports: []corev1.ContainerPort{ @@ -436,7 +438,24 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam Name: "CLOUD_GAME_WEBRTC_ICESERVERS_1_USERNAME", Value: gatewayConfig.Spec.UserName, }, - }, //TODO mount game executable and config game config file + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "gamestorage", + MountPath: fullpath, + SubPath: "varooom-3d (1).gba", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "gamestorage", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "azure-blob-pvc", + }, + }, }, }, }, @@ -472,7 +491,7 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r Containers: []corev1.Container{ { Name: "worker", - Image: "ghcr.io/giongto35/cloud-game/cloud-game:master", + Image: "ghcr.io/giongto35/cloud-game/cloud-game:v3.0.5", Command: []string{"./worker"}, Args: []string{""}, Ports: []corev1.ContainerPort{ From 01983728b21358bacc19cd63760eb4b528fe8f96 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Tue, 18 Jun 2024 13:54:18 +0200 Subject: [PATCH 23/25] Add FileName to operator and api crd --- api/apis/k8sApi.go | 5 +++-- operator/api/stream/v1/game_types.go | 5 ++--- .../crd/bases/stream.indiegamestream.com_games.yaml | 6 ++---- operator/config/samples/stream_v1_game.yaml | 2 +- operator/internal/controller/stream/game_controller.go | 10 +++++----- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/api/apis/k8sApi.go b/api/apis/k8sApi.go index 7b567a3..a51a26e 100644 --- a/api/apis/k8sApi.go +++ b/api/apis/k8sApi.go @@ -4,6 +4,7 @@ import ( "api/models" "context" "errors" + "github.com/google/uuid" streamv1 "indiegamestream.com/indiegamestream/api/stream/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -87,8 +88,8 @@ func createAndVerifyGameResource(game *models.Game) (*streamv1.Game, error) { Namespace: "default", }, Spec: streamv1.GameSpec{ - Name: game.Title, - ExecutableURL: game.StorageLocation, + Name: game.Title, + FileName: game.FileName, }, }, nil } diff --git a/operator/api/stream/v1/game_types.go b/operator/api/stream/v1/game_types.go index 5636035..0ddbd33 100644 --- a/operator/api/stream/v1/game_types.go +++ b/operator/api/stream/v1/game_types.go @@ -26,9 +26,8 @@ import ( // GameSpec defines the desired state of Game type GameSpec struct { // Name of the game - Name string `json:"name"` - // ExecutableURL is the URL of the game executable, which will be downloaded and mounted into the container - ExecutableURL string `json:"executableURL"` + Name string `json:"name"` + FileName string `json:"filename"` } // GameStatus defines the observed state of Game diff --git a/operator/config/crd/bases/stream.indiegamestream.com_games.yaml b/operator/config/crd/bases/stream.indiegamestream.com_games.yaml index 4d72e5f..c5e3923 100644 --- a/operator/config/crd/bases/stream.indiegamestream.com_games.yaml +++ b/operator/config/crd/bases/stream.indiegamestream.com_games.yaml @@ -43,15 +43,13 @@ spec: spec: description: GameSpec defines the desired state of Game properties: - executableURL: - description: ExecutableURL is the URL of the game executable, which - will be downloaded and mounted into the container + filename: type: string name: description: Name of the game type: string required: - - executableURL + - filename - name type: object status: diff --git a/operator/config/samples/stream_v1_game.yaml b/operator/config/samples/stream_v1_game.yaml index 9aecd74..f6b5065 100644 --- a/operator/config/samples/stream_v1_game.yaml +++ b/operator/config/samples/stream_v1_game.yaml @@ -7,4 +7,4 @@ metadata: name: game-sample2 spec: name: "SPECGAME2" - executableURL: "https://example.com/game" + fileName: "varooom-3d (1).gba" diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index aec32fb..f9342ca 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -78,7 +78,7 @@ func (r *GameReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{}, err } - log.Info("Reconciling Game", "Name", game.Spec.Name, "ExecutableURL", game.Spec.ExecutableURL) + log.Info("Reconciling Game", "Name", game.Spec.Name, "FileName", game.Spec.FileName) // name of our custom finalizer gameFinalizer := "game.stream.indiegamestream.com/finalizer" @@ -385,7 +385,7 @@ func int32Ptr(i int32) *int32 { } func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Game, resourceName string, gatewayConfig *stunnerv1.GatewayConfig, gatewayIP string) (*appsv1.Deployment, error) { - fullpath := fmt.Sprintf("/usr/local/share/cloud-game/assets/games/%s", "test.gba") + fullpath := fmt.Sprintf("/usr/local/share/cloud-game/assets/games/%s", game.Spec.FileName) dep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -443,7 +443,7 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam { Name: "gamestorage", MountPath: fullpath, - SubPath: "varooom-3d (1).gba", + SubPath: game.Spec.FileName, }, }, }, @@ -471,7 +471,7 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam } func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, resourceName string, coordIP string, workerIP string) (*appsv1.Deployment, error) { - fullpath := fmt.Sprintf("/usr/local/share/cloud-game/assets/games/%s", "test.gba") + fullpath := fmt.Sprintf("/usr/local/share/cloud-game/assets/games/%s", game.Spec.FileName) dep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -517,7 +517,7 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r { Name: "gamestorage", MountPath: fullpath, - SubPath: "varooom-3d (1).gba", + SubPath: game.Spec.FileName, }, }, }, From 498e9c7b9e67f54e3283d64e3e7763cc555739ad Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Tue, 18 Jun 2024 15:35:49 +0200 Subject: [PATCH 24/25] Fix weird service bug --- .../controller/stream/game_controller.go | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index f9342ca..e425bc0 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "strings" "time" "k8s.io/apimachinery/pkg/api/errors" @@ -386,6 +387,7 @@ func int32Ptr(i int32) *int32 { func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Game, resourceName string, gatewayConfig *stunnerv1.GatewayConfig, gatewayIP string) (*appsv1.Deployment, error) { fullpath := fmt.Sprintf("/usr/local/share/cloud-game/assets/games/%s", game.Spec.FileName) + newSelector := fmt.Sprintf("%s-%s", "coordinator", game.Name) dep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -395,11 +397,11 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam Spec: appsv1.DeploymentSpec{ Replicas: int32Ptr(1), Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "coordinator"}, + MatchLabels: map[string]string{"app": newSelector}, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "coordinator"}, + Labels: map[string]string{"app": newSelector}, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -443,7 +445,7 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam { Name: "gamestorage", MountPath: fullpath, - SubPath: game.Spec.FileName, + SubPath: game.Name, }, }, }, @@ -472,7 +474,7 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, resourceName string, coordIP string, workerIP string) (*appsv1.Deployment, error) { fullpath := fmt.Sprintf("/usr/local/share/cloud-game/assets/games/%s", game.Spec.FileName) - + newSelector := fmt.Sprintf("%s-%s", "worker", game.Name) dep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -481,11 +483,11 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r Spec: appsv1.DeploymentSpec{ Replicas: int32Ptr(2), Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "worker"}, + MatchLabels: map[string]string{"app": newSelector}, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "worker"}, + Labels: map[string]string{"app": newSelector}, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -517,7 +519,7 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r { Name: "gamestorage", MountPath: fullpath, - SubPath: game.Spec.FileName, + SubPath: game.Name, }, }, }, @@ -548,9 +550,10 @@ func (r *GameReconciler) constructLoadBalancer(game *streamv1.Game, name string, className := "tailscale" annotation := fmt.Sprintf("%s-%s", game.Spec.Name, game.Name) + newSelector := fmt.Sprintf("%s-%s", selector, game.Name) annotations := map[string]string{} outsidePort := port - if selector == "coordinator" { + if strings.HasPrefix(selector, "coordinator") { annotations["tailscale.com/hostname"] = annotation outsidePort = 80 } @@ -562,7 +565,7 @@ func (r *GameReconciler) constructLoadBalancer(game *streamv1.Game, name string, Annotations: annotations, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{"app": selector}, + Selector: map[string]string{"app": newSelector}, LoadBalancerClass: &className, Ports: []corev1.ServicePort{ { @@ -581,13 +584,14 @@ func (r *GameReconciler) constructLoadBalancer(game *streamv1.Game, name string, return svc, nil } func (r *GameReconciler) constructLoadBalancerUDP(game *streamv1.Game, name string, selector string, port int32) (*corev1.Service, error) { + newSelector := fmt.Sprintf("%s-%s", "worker", game.Name) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: game.Namespace, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{"app": selector}, + Selector: map[string]string{"app": newSelector}, Ports: []corev1.ServicePort{ { Protocol: corev1.ProtocolUDP, From 1770dd44d338329fbdb50df522fe76137340c8c3 Mon Sep 17 00:00:00 2001 From: Jonas Konrad Date: Tue, 18 Jun 2024 15:36:05 +0200 Subject: [PATCH 25/25] Use different storage account for kubernetes --- .github/workflows/deploy-az.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 548a406..4110217 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -64,8 +64,8 @@ jobs: - name: Initialize CSI working-directory: ./scripts/azure-csi run: | - STORAGE_KEY=$(az storage account keys list --resource-group rg-management-not2day --account-name ${{secrets.AZURERM_GAME_STORAGE_ACCOUNT_NAME}} --query "[0].value" -o tsv) - kubectl create secret generic azure-secret --from-literal=azurestorageaccountname=${{secrets.AZURERM_GAME_STORAGE_ACCOUNT_NAME}} --from-literal=azurestorageaccountkey=$STORAGE_KEY + STORAGE_KEY=$(az storage account keys list --resource-group rg-management-not2day --account-name ${{secrets.AZURERM_STORAGE_ACCOUNT_NAME}} --query "[0].value" -o tsv) + kubectl create secret generic azure-secret --from-literal=azurestorageaccountname=${{secrets.AZURERM_STORAGE_ACCOUNT_NAME}} --from-literal=azurestorageaccountkey=$STORAGE_KEY kubectl apply -f initialize-csi-storage.yaml - name: Initialize Cluster permissions