diff --git a/.github/workflows/deploy-az.yml b/.github/workflows/deploy-az.yml index 689fcd3..548a406 100644 --- a/.github/workflows/deploy-az.yml +++ b/.github/workflows/deploy-az.yml @@ -47,8 +47,7 @@ jobs: - name: Apply tailscale operator working-directory: ./iac run: | - az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --namespace=tailscale --create-namespace --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait" - + az aks command invoke -n ${{ secrets.AZURERM_AKS_CLUSTER_NAME }} -g rg-service-not2day --command "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update && helm upgrade --install tailscale-operator tailscale/tailscale-operator --set-string oauth.clientId=${{secrets.TAILSCALE_CLIENT_ID}} --set-string oauth.clientSecret=${{secrets.TAILSCALE_CLIENT_SECRET}} --set-string apiServerProxyConfig.mode=true --wait || true" - name: Connect to tailscale uses: tailscale/github-action@v2 with: @@ -62,6 +61,105 @@ jobs: - name: Check working cluster run: kubectl get pods -A + - name: Initialize CSI + working-directory: ./scripts/azure-csi + run: | + STORAGE_KEY=$(az storage account keys list --resource-group rg-management-not2day --account-name ${{secrets.AZURERM_GAME_STORAGE_ACCOUNT_NAME}} --query "[0].value" -o tsv) + kubectl create secret generic azure-secret --from-literal=azurestorageaccountname=${{secrets.AZURERM_GAME_STORAGE_ACCOUNT_NAME}} --from-literal=azurestorageaccountkey=$STORAGE_KEY + kubectl apply -f initialize-csi-storage.yaml + + - name: Initialize Cluster permissions + working-directory: ./scripts/cluster-permissions + run: | + kubectl apply -f api_cluster_permission.yaml + + - name: Install MySQL + working-directory: ./helm/mysql + run: | + helm repo add mysql-operator https://mysql.github.io/mysql-operator/ + helm repo update + helm install mysql-operator mysql-operator/mysql-operator --version "2.1.3" --wait \ + --create-namespace --namespace=mysql-operator || true + helm install mysql mysql-operator/mysql-innodbcluster --version "2.1.3" --wait \ + --create-namespace --namespace=mysql -f values.yaml \ + --set-string credentials.root.password=${{ secrets.MYSQL_ROOT_PASSWORD }} || true + + - name: Install Open Policy Gatekeeper + working-directory: ./scripts/opa + run: | + helm repo add gatekeeper https://open-policy-agent.github.io/gatekeeper/charts + helm repo update + helm install gatekeeper gatekeeper/gatekeeper --namespace gatekeeper-system --create-namespace --wait || true + kubectl apply -f loadbalancerclass_mutator.yaml + + - name: Install stunner + working-directory: ./scripts/localenv + run: make install_stunner || true + + - name: Install game operator manifests + working-directory: ./operator + run: make install + + - name: Deploy game operator + working-directory: ./operator + run: make deploy IMG=${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.SUB_NAMESPACE }}/operator:${{ env.LABEL }} + + # - name: Wait for MySQL to be ready + # run: | + # while true; do + # POD_STATUS=$(kubectl get pod mysql-0 -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); + # if [ "$POD_STATUS" ]; then + # echo "Pod mysql-0 has been created with status: $POD_STATUS"; + # break; + # else + # echo "Waiting for pod mysql-0 to be created..."; + # sleep 5; + # fi + # done + # kubectl wait --for=condition=Ready pod/mysql-0 -n mysql --timeout=120s + + # while true; do + # POD_STATUS=$(kubectl get pod -l app.kubernetes.io/component=router -n mysql --no-headers -o custom-columns=":status.phase" 2>/dev/null); \ + # if [ "$POD_STATUS" ]; then + # echo "MySQL router has been created with status: $POD_STATUS"; + # break; + # else + # echo "Waiting for MySQL router to be created..."; + # sleep 5; + # fi + # done + # kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=router -n mysql --timeout=120s + + # - name: Install API + # working-directory: ./helm/api + # run: | + # helm install -f values.yaml \ + # --set-string env.mysqlRootPassword=${{ secrets.MYSQL_ROOT_PASSWORD }} \ + # --set-string env.azureTenantId=${{ secrets.AZURE_TENANT_ID }} + # --set-string env.azureClientId=${{ secrets.CLIENT_ID }} + # --set-string env.azureClientSecret=${{ secrets.CLIENT_SECRET }} + # --set-string env.azureStorageAccount=${{ secrets.AZURERM_STORAGE_ACCOUNT_NAME }} + # --set-string env.azureContainerName=${{ secrets.AZURERM_CONTAINER_NAME }} + # --set-string env.azureAksClusterName=${{ secrets.AZURERM_AKS_CLUSTER_NAME }} + # --set-string env.azurermSubscriptionId=${{ secrets.AZURERM_SUBSCRIPTION_ID }} + # --set-string env.azurermResourceGroupName=${{ secrets.AZURERM_RESOURCE_GROUP_NAME }} + # --set-string image.label=${{ env.LABEL }} \ + # api . + + # - name: Wait for external IP of API + # run: | + # until [ -n "$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do + # sleep 5 + # done + + # - name: Install frontend + # working-directory: ./helm/frontend + # run: | + # helm install -f values.yaml \ + # --set-string appConfig.apiUrl=http://$(kubectl get svc api -n api -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):$(kubectl get svc api -n api -o jsonpath='{.spec.ports[0].port}') \ + # --set-string image.label=${{ env.LABEL }} \ + # frontend . + - name: Logout of Azure run: az logout @@ -84,6 +182,9 @@ jobs: - name: Login to Azure run: az login --service-principal -u ${{ secrets.CLIENT_ID }} -p ${{ secrets.CLIENT_SECRET }} --tenant ${{ secrets.AZURERM_TENANT_ID }} + + + - name: Terraform Apply working-directory: ./iac run: | diff --git a/api/apis/k8sApi.go b/api/apis/k8sApi.go index 7b567a3..a51a26e 100644 --- a/api/apis/k8sApi.go +++ b/api/apis/k8sApi.go @@ -4,6 +4,7 @@ import ( "api/models" "context" "errors" + "github.com/google/uuid" streamv1 "indiegamestream.com/indiegamestream/api/stream/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -87,8 +88,8 @@ func createAndVerifyGameResource(game *models.Game) (*streamv1.Game, error) { Namespace: "default", }, Spec: streamv1.GameSpec{ - Name: game.Title, - ExecutableURL: game.StorageLocation, + Name: game.Title, + FileName: game.FileName, }, }, nil } diff --git a/api/migrations/2_game_filename.sql b/api/migrations/2_game_filename.sql new file mode 100644 index 0000000..4df0342 --- /dev/null +++ b/api/migrations/2_game_filename.sql @@ -0,0 +1,2 @@ +ALTER TABLE games ADD FileName varchar(512); +INSERT INTO db_state VALUES (2); \ No newline at end of file diff --git a/api/models/game.go b/api/models/game.go index 5ea2fc1..63a6e19 100644 --- a/api/models/game.go +++ b/api/models/game.go @@ -11,5 +11,6 @@ type Game struct { StorageLocation string `json:"storageLocation"` Status shared.GameStatus `json:"status"` Url string `json:"url"` - Owner string `json:"owner"` -} + Owner string `json:"owner"` + FileName string `json:"fileName"` +} diff --git a/api/repositories/gameRepository.go b/api/repositories/gameRepository.go index a2b6b24..34cb212 100644 --- a/api/repositories/gameRepository.go +++ b/api/repositories/gameRepository.go @@ -52,7 +52,7 @@ func (g gameRepository) FindAllByOwner(owner string) ([]models.Game, error) { func (g gameRepository) FindByID(id uuid.UUID) (*models.Game, error) { var game models.Game err := g.db.QueryRow("SELECT * FROM games WHERE ID = ?", id). - Scan(&game.ID, &game.Title, &game.StorageLocation, &game.Status, &game.Url, &game.Owner) + Scan(&game.ID, &game.Title, &game.StorageLocation, &game.Status, &game.Url, &game.Owner, &game.FileName) if err != nil { if err == sql.ErrNoRows { return nil, nil @@ -74,24 +74,24 @@ func (g gameRepository) Save(game *models.Game) error { if existing != nil { //If yes, update the existing entry - stmt, err := g.db.Prepare("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=? WHERE ID = ?") + stmt, err := g.db.Prepare("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=?, FileName=? WHERE ID = ?") if err != nil { return err } - return checkResult(stmt.Exec(game.Title, game.StorageLocation, game.Status, game.Url, game.ID)) + return checkResult(stmt.Exec(game.Title, game.StorageLocation, game.Status, game.Url, game.ID, game.FileName)) } } else { game.ID = uuid.New() } //If not create a new one - stmt, err := g.db.Prepare("INSERT INTO games (ID, Title, StorageLocation, Status, Url, Owner) VALUES (?,?,?,?,?,?)") + stmt, err := g.db.Prepare("INSERT INTO games (ID, Title, StorageLocation, Status, Url, Owner, FileName) VALUES (?,?,?,?,?,?,?)") if err != nil { return err } - return checkResult(stmt.Exec(game.ID, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner)) + return checkResult(stmt.Exec(game.ID, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner, game.FileName)) } // Delete removes the entry with a specific id from the games database. @@ -128,7 +128,7 @@ func readGamesFromRows(query *sql.Rows) ([]models.Game, error) { var games = []models.Game{} for query.Next() { var game models.Game - err := query.Scan(&game.ID, &game.Title, &game.StorageLocation, &game.Status, &game.Url, &game.Owner) + err := query.Scan(&game.ID, &game.Title, &game.StorageLocation, &game.Status, &game.Url, &game.Owner, &game.FileName) if err != nil { return nil, err } diff --git a/api/services/gameService.go b/api/services/gameService.go index 53b7814..b24db50 100644 --- a/api/services/gameService.go +++ b/api/services/gameService.go @@ -54,6 +54,7 @@ func (g gameService) Save(fileHeader *multipart.FileHeader, title string, owner Status: shared.Status_New, Url: "", Owner: owner, + FileName: fileHeader.Filename, } //Upload game to azure blob storage container diff --git a/api/tests/gameRepository_test.go b/api/tests/gameRepository_test.go index 42e1d16..e4719d9 100644 --- a/api/tests/gameRepository_test.go +++ b/api/tests/gameRepository_test.go @@ -88,10 +88,11 @@ func Test_Create_Game_Without_Id_Should_Succeed_And_SetId(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectPrepare(regexp.QuoteMeta("INSERT INTO games")) mock.ExpectExec(regexp.QuoteMeta("INSERT INTO games")). - WithArgs(sqlmock.AnyArg(), game.Title, game.StorageLocation, game.Status, game.Url, game.Owner). + WithArgs(sqlmock.AnyArg(), game.Title, game.StorageLocation, game.Status, game.Url, game.Owner, game.FileName). WillReturnResult(sqlmock.NewResult(0, 1)) //Run the test @@ -128,6 +129,7 @@ func Test_Create_Game_With_Id_Should_Succeed(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM games WHERE ID = ?")). @@ -136,7 +138,7 @@ func Test_Create_Game_With_Id_Should_Succeed(t *testing.T) { mock.ExpectPrepare("INSERT INTO games") mock.ExpectExec("INSERT INTO games"). - WithArgs(game.ID, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner). + WithArgs(game.ID, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner, game.FileName). WillReturnResult(sqlmock.NewResult(0, 1)) //Run the test @@ -176,20 +178,21 @@ func Test_Save_Existing_Game_Should_Succeed(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM games WHERE ID = ?")). WithArgs(id).WillReturnRows( - sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner"}). - AddRow(id, "", "", "", "", ""), + sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner", "FileName"}). + AddRow(id, "", "", "", "", "", ""), ) mock.ExpectPrepare(regexp. - QuoteMeta("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=? WHERE ID = ?")) + QuoteMeta("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=?, FileName=? WHERE ID = ?")) mock.ExpectExec(regexp. - QuoteMeta("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=? WHERE ID = ?")). - WithArgs(game.Title, game.StorageLocation, game.Status, game.Url, game.ID). + QuoteMeta("UPDATE games SET Title=?, StorageLocation=?, Status=?, Url=?, FileName=? WHERE ID = ?")). + WithArgs(game.Title, game.StorageLocation, game.Status, game.Url, game.ID, game.FileName). WillReturnResult(sqlmock.NewResult(0, 1)) //Run the test @@ -229,12 +232,13 @@ func Test_Find_Game_By_Id_Should_Succeed(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM games WHERE ID = ?")). WithArgs(id).WillReturnRows( - sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner"}). - AddRow(id, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner), + sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner", "FileName"}). + AddRow(id, game.Title, game.StorageLocation, game.Status, game.Url, game.Owner, game.FileName), ) //Run the test @@ -295,6 +299,7 @@ func Test_Find_Two_Games_Of_Same_Owner_Should_Succeed(t *testing.T) { Status: "AMockC", Url: "AMockD", Owner: "MockOwner", + FileName: "TestFile.nes", } gameB := models.Game{ @@ -304,15 +309,16 @@ func Test_Find_Two_Games_Of_Same_Owner_Should_Succeed(t *testing.T) { Status: "BMockC", Url: "BMockD", Owner: "MockOwner", + FileName: "TestFile2.nes", } mock.ExpectPrepare(regexp.QuoteMeta("SELECT * FROM games WHERE owner = ?")) mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM games WHERE owner = ?")). WithArgs("MockOwner"). WillReturnRows( - sqlmock.NewRows([]string{"ID", "Title", "StorageLocation", "Status", "Url", "Owner"}). - AddRow(gameA.ID, gameA.Title, gameA.StorageLocation, gameA.Status, gameA.Url, gameA.Owner). - AddRow(gameB.ID, gameB.Title, gameB.StorageLocation, gameB.Status, gameB.Url, gameB.Owner), + sqlmock.NewRows([]string{"ID", "Title", "StorageLocation", "Status", "Url", "Owner", "FileName"}). + AddRow(gameA.ID, gameA.Title, gameA.StorageLocation, gameA.Status, gameA.Url, gameA.Owner, gameA.FileName). + AddRow(gameB.ID, gameB.Title, gameB.StorageLocation, gameB.Status, gameB.Url, gameB.Owner, gameB.FileName), ) //Run the test @@ -350,7 +356,7 @@ func Test_Find_Games_When_Database_Is_Empty_Should_Succeed(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM games WHERE owner = ?")). WithArgs("MockOwner"). WillReturnRows( - sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner"}), + sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner", "FileName"}), ) //Run the test @@ -389,11 +395,12 @@ func Test_Read_Owner_Should_Succeed(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectQuery(regexp.QuoteMeta("SELECT Owner FROM games WHERE ID = ?")). WithArgs(id). - WillReturnRows(sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner"})) + WillReturnRows(sqlmock.NewRows([]string{"Id", "Title", "StorageLocation", "Status", "Url", "Owner", "FileName"})) //Run the test repository := repositories.GameRepository(db) @@ -427,6 +434,7 @@ func Test_Read_Owner_Should_Throw_When_Database_Is_Empty(t *testing.T) { Status: "MockStatus", Url: "MockUrl", Owner: "MockOwner", + FileName: "TestFile.nes", } mock.ExpectQuery(regexp.QuoteMeta("SELECT Owner FROM games WHERE ID = ?")). @@ -483,4 +491,8 @@ func compareGames(t *testing.T, game *models.Game, res *models.Game) { t.Errorf("game owner has been changed") } + if game.FileName != res.FileName { + t.Errorf("game file name has been changed") + } + } diff --git a/operator/api/stream/v1/game_types.go b/operator/api/stream/v1/game_types.go index 5636035..0ddbd33 100644 --- a/operator/api/stream/v1/game_types.go +++ b/operator/api/stream/v1/game_types.go @@ -26,9 +26,8 @@ import ( // GameSpec defines the desired state of Game type GameSpec struct { // Name of the game - Name string `json:"name"` - // ExecutableURL is the URL of the game executable, which will be downloaded and mounted into the container - ExecutableURL string `json:"executableURL"` + Name string `json:"name"` + FileName string `json:"filename"` } // GameStatus defines the observed state of Game diff --git a/operator/config/crd/bases/stream.indiegamestream.com_games.yaml b/operator/config/crd/bases/stream.indiegamestream.com_games.yaml index 4d72e5f..c5e3923 100644 --- a/operator/config/crd/bases/stream.indiegamestream.com_games.yaml +++ b/operator/config/crd/bases/stream.indiegamestream.com_games.yaml @@ -43,15 +43,13 @@ spec: spec: description: GameSpec defines the desired state of Game properties: - executableURL: - description: ExecutableURL is the URL of the game executable, which - will be downloaded and mounted into the container + filename: type: string name: description: Name of the game type: string required: - - executableURL + - filename - name type: object status: diff --git a/operator/config/rbac/kustomization.yaml b/operator/config/rbac/kustomization.yaml index 731832a..fc4f042 100644 --- a/operator/config/rbac/kustomization.yaml +++ b/operator/config/rbac/kustomization.yaml @@ -9,6 +9,8 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +- game_editor_role.yaml +- game_viewer_role.yaml # Comment the following 4 lines if you want to disable # the auth proxy (https://github.com/brancz/kube-rbac-proxy) # which protects your /metrics endpoint. diff --git a/operator/config/samples/stream_v1_game.yaml b/operator/config/samples/stream_v1_game.yaml index 9aecd74..f6b5065 100644 --- a/operator/config/samples/stream_v1_game.yaml +++ b/operator/config/samples/stream_v1_game.yaml @@ -7,4 +7,4 @@ metadata: name: game-sample2 spec: name: "SPECGAME2" - executableURL: "https://example.com/game" + fileName: "varooom-3d (1).gba" diff --git a/operator/internal/controller/stream/game_controller.go b/operator/internal/controller/stream/game_controller.go index 316177b..f9342ca 100644 --- a/operator/internal/controller/stream/game_controller.go +++ b/operator/internal/controller/stream/game_controller.go @@ -78,7 +78,7 @@ func (r *GameReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{}, err } - log.Info("Reconciling Game", "Name", game.Spec.Name, "ExecutableURL", game.Spec.ExecutableURL) + log.Info("Reconciling Game", "Name", game.Spec.Name, "FileName", game.Spec.FileName) // name of our custom finalizer gameFinalizer := "game.stream.indiegamestream.com/finalizer" @@ -196,7 +196,13 @@ func (r *GameReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. // Note that Status is a subresource, so changes to it are ignored by the cache, hence the need to update it manually //game.Status.Nodes = nodes //game.Status.Phase = phase - game.Status.URL = fmt.Sprintf("http://%s:8000", coordIP) + outsidehostname, err := waitForLoadBalancerHostname(ctx, r.Client, game.Namespace, coordinatorName) + if err != nil { + log.Error(err, "unable to get LoadBalancer Hostname for Coordinator") + return ctrl.Result{}, err + } + + game.Status.URL = fmt.Sprintf("http://%s", outsidehostname) //TODO: Add nginx ingress url to status if err := r.Status().Update(ctx, game); err != nil { log.Error(err, "unable to update Game status") @@ -379,6 +385,8 @@ func int32Ptr(i int32) *int32 { } func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Game, resourceName string, gatewayConfig *stunnerv1.GatewayConfig, gatewayIP string) (*appsv1.Deployment, error) { + fullpath := fmt.Sprintf("/usr/local/share/cloud-game/assets/games/%s", game.Spec.FileName) + dep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -397,9 +405,9 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam Containers: []corev1.Container{ { Name: "coordinator", - Image: "valniae/snekyrepo:crdi", - Command: []string{"coordinator"}, - Args: []string{"--v=5"}, + Image: "ghcr.io/giongto35/cloud-game/cloud-game:v3.0.5", + Command: []string{"./coordinator"}, + Args: []string{""}, Ports: []corev1.ContainerPort{ { ContainerPort: 8000, @@ -411,7 +419,7 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam Value: gatewayConfig.Spec.Password, }, { - Name: "CLOUD_GAME_WEBRTC_ICESERVERS_0_URL", + Name: "CLOUD_GAME_WEBRTC_ICESERVERS_0_URLS", Value: fmt.Sprintf("turn:%s:3478", gatewayIP), }, { @@ -423,14 +431,31 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam Value: gatewayConfig.Spec.Password, }, { - Name: "CLOUD_GAME_WEBRTC_ICESERVERS_1_URL", + Name: "CLOUD_GAME_WEBRTC_ICESERVERS_1_URLS", Value: fmt.Sprintf("turn:%s:3478", gatewayIP), }, { Name: "CLOUD_GAME_WEBRTC_ICESERVERS_1_USERNAME", Value: gatewayConfig.Spec.UserName, }, - }, //TODO mount game executable and config game config file + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "gamestorage", + MountPath: fullpath, + SubPath: game.Spec.FileName, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "gamestorage", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "azure-blob-pvc", + }, + }, }, }, }, @@ -446,6 +471,8 @@ func (r *GameReconciler) constructControllerDeploymentForGame(game *streamv1.Gam } func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, resourceName string, coordIP string, workerIP string) (*appsv1.Deployment, error) { + fullpath := fmt.Sprintf("/usr/local/share/cloud-game/assets/games/%s", game.Spec.FileName) + dep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -464,9 +491,9 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r Containers: []corev1.Container{ { Name: "worker", - Image: "valniae/snekyrepo:crdi", - Command: []string{"worker"}, - Args: []string{"--v=5"}, + Image: "ghcr.io/giongto35/cloud-game/cloud-game:v3.0.5", + Command: []string{"./worker"}, + Args: []string{""}, Ports: []corev1.ContainerPort{ { ContainerPort: 8443, @@ -479,13 +506,30 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r }, { Name: "CLOUD_GAME_WORKER_NETWORK_COORDINATORADDRESS", - Value: fmt.Sprintf("%s:8000", coordIP), + Value: fmt.Sprintf("%s:80", coordIP), }, { Name: "CLOUD_GAME_WORKER_NETWORK_PUBLICADDRESS", Value: workerIP, }, - }, //TODO mount game executable and config game config file + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "gamestorage", + MountPath: fullpath, + SubPath: game.Spec.FileName, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "gamestorage", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "azure-blob-pvc", + }, + }, }, }, }, @@ -503,18 +547,26 @@ func (r *GameReconciler) constructWorkerDeploymentForGame(game *streamv1.Game, r func (r *GameReconciler) constructLoadBalancer(game *streamv1.Game, name string, selector string, port int32) (*corev1.Service, error) { className := "tailscale" + annotation := fmt.Sprintf("%s-%s", game.Spec.Name, game.Name) + annotations := map[string]string{} + outsidePort := port + if selector == "coordinator" { + annotations["tailscale.com/hostname"] = annotation + outsidePort = 80 + } svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: game.Namespace, + Name: name, + Namespace: game.Namespace, + Annotations: annotations, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{"app": selector}, LoadBalancerClass: &className, Ports: []corev1.ServicePort{ { - Port: port, + Port: outsidePort, TargetPort: intstr.FromInt32(port), }, }, @@ -583,6 +635,41 @@ func waitForLoadBalancerIP(ctx context.Context, k8sClient client.Client, namespa return ip, nil } +func waitForLoadBalancerHostname(ctx context.Context, k8sClient client.Client, namespace, serviceName string) (string, error) { + var hostname string + + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 20*time.Second, true, func(ctx context.Context) (bool, error) { + svc := &corev1.Service{} + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: serviceName}, svc); err != nil { + return false, err + } + if len(svc.Status.LoadBalancer.Ingress) > 0 { + hostname = svc.Status.LoadBalancer.Ingress[0].Hostname + if hostname != "" { + return true, nil + } + if len(svc.Status.LoadBalancer.Ingress) > 1 { + hostname = svc.Status.LoadBalancer.Ingress[1].Hostname + if hostname != "" { + return true, nil + } + } + } + return false, nil + }) + + //fallback on ip + if err != nil { + ip, err := waitForLoadBalancerIP(ctx, k8sClient, namespace, serviceName) + if err != nil { + return "", err + } + return ip, nil + } + + return hostname, nil +} + var ( udpRouteOwnerKey = ".metadata.controller" apiGVStr = streamv1.GroupVersion.String() diff --git a/scripts/azure-csi/initialize-csi-storage.yaml b/scripts/azure-csi/initialize-csi-storage.yaml new file mode 100644 index 0000000..80963dc --- /dev/null +++ b/scripts/azure-csi/initialize-csi-storage.yaml @@ -0,0 +1,37 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azureblob-sc +provisioner: blob.csi.azure.com +parameters: + containerName: games + storageAccount: indiegamestream + csi.storage.k8s.io/provisioner-secret-name: azure-secret + csi.storage.k8s.io/provisioner-secret-namespace: default + csi.storage.k8s.io/node-stage-secret-name: azure-secret + csi.storage.k8s.io/node-stage-secret-namespace: default +reclaimPolicy: Retain +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: azure-blob-pvc +spec: + accessModes: + - ReadOnlyMany + storageClassName: azureblob-sc + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/scripts/cluster-permissions/api_cluster_permission.yaml b/scripts/cluster-permissions/api_cluster_permission.yaml new file mode 100644 index 0000000..7c6c4d2 --- /dev/null +++ b/scripts/cluster-permissions/api_cluster_permission.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: game-editor-binding +subjects: +- kind: ServiceAccount + name: default + namespace: api +roleRef: + kind: ClusterRole + name: game-editor-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file