Fix/gcp helm agent

.tools/nvim/__http__/infra/clusters.graphql.yml

@@ -277,8 +277,8 @@ query: |+ #graphql
     infra_upgradeHelmKloudliteAgent(clusterName: $clusterName)
   }
 variables:
-  # clusterName: "{{.clusterName}}"
-  clusterName: "{{.gcpClusterName}}"
+  clusterName: "{{.clusterName}}"
+  # clusterName: "{{.gcpClusterName}}"
 ---
 
 label: Delete Cluster

Taskfile.yml

@@ -10,7 +10,7 @@ tasks:
 
           banner="// Code generated by mocki. DO NOT EDIT."
 
-          mock_module_path=$(echo $package_path | sed 's/kloudlite.io\///')
+          mock_module_path=$(echo $package_path | sed 's/github.com\/kloudlite\/api\///')
           file_name=$(echo $interface_name | awk '{print tolower($0)}')
           file_name+=".go"
 
@@ -26,39 +26,43 @@ tasks:
 
           printf "mocking $package_path.$interface_name "
           mkdir -p mocks/$mock_module_path
+          set -x 
+          whereis mocki
           mocki --package $package_path --interface $interface_name > mocks/$mock_module_path/$file_name
+          set +x
           echo "✅"
         }
 
-        gen_mock kloudlite.io/pkg/repos DbRepo
+        gen_mock github.com/kloudlite/api/pkg/repos DbRepo
 
-        gen_mock kloudlite.io/pkg/k8s Client
-        gen_mock kloudlite.io/apps/infra/internal/domain AccountsSvc
+        gen_mock github.com/kloudlite/api/pkg/k8s Client
 
-        gen_mock kloudlite.io/grpc-interfaces/kloudlite.io/rpc/iam IAMClient
-        gen_mock kloudlite.io/grpc-interfaces/kloudlite.io/rpc/accounts AccountsClient
-        gen_mock kloudlite.io/grpc-interfaces/kloudlite.io/rpc/message-office-internal MessageOfficeInternalClient
-
-        gen_mock kloudlite.io/pkg/kafka Producer
-        gen_mock kloudlite.io/pkg/kafka Consumer
+        # gen_mock github.com/kloudlite/api/apps/infra/internal/domain AccountsSvc
+        #
+        # gen_mock github.com/kloudlite/api/grpc-interfaces/github.com/kloudlite/api/rpc/iam IAMClient
+        # gen_mock github.com/kloudlite/api/grpc-interfaces/github.com/kloudlite/api/rpc/accounts AccountsClient
+        # gen_mock github.com/kloudlite/api/grpc-interfaces/github.com/kloudlite/api/rpc/message-office-internal MessageOfficeInternalClient
+        #
+        # gen_mock github.com/kloudlite/api/pkg/kafka Producer
+        # gen_mock github.com/kloudlite/api/pkg/kafka Consumer
 
       # - mkdir -p pkg/repos/mocks
-      # - mocki --package kloudlite.io/pkg/repos --interface DbRepo > pkg/repos/mocks/db-repo.go
+      # - mocki --package github.com/kloudlite/api/pkg/repos --interface DbRepo > pkg/repos/mocks/db-repo.go
       #
       # - mkdir -p pkg/k8s/mocks
-      # - mocki --package kloudlite.io/pkg/k8s --interface ExtendedK8sClient  > pkg/k8s/mocks/extended-k8s-client.go
+      # - mocki --package github.com/kloudlite/api/pkg/k8s --interface ExtendedK8sClient  > pkg/k8s/mocks/extended-k8s-client.go
       #
       # # grpc-interfaces mocks
-      # - mkdir -p mocks/grpc-interfaces/kloudlite.io/rpc/iam
-      # - mocki --package kloudlite.io/grpc-interfaces/kloudlite.io/rpc/iam --interface IAMClient > mocks/grpc-interfaces/kloudlite.io/rpc/iam/iam-client.go
+      # - mkdir -p mocks/grpc-interfaces/github.com/kloudlite/api/rpc/iam
+      # - mocki --package github.com/kloudlite/api/grpc-interfaces/github.com/kloudlite/api/rpc/iam --interface IAMClient > mocks/grpc-interfaces/github.com/kloudlite/api/rpc/iam/iam-client.go
       #
-      # - mkdir -p mocks/grpc-interfaces/kloudlite.io/rpc/accounts
-      # - mocki --package kloudlite.io/grpc-interfaces/kloudlite.io/rpc/accounts --interface AccountsClient > mocks/grpc-interfaces/kloudlite.io/rpc/accounts/accounts-client.go
+      # - mkdir -p mocks/grpc-interfaces/github.com/kloudlite/api/rpc/accounts
+      # - mocki --package github.com/kloudlite/api/grpc-interfaces/github.com/kloudlite/api/rpc/accounts --interface AccountsClient > mocks/grpc-interfaces/github.com/kloudlite/api/rpc/accounts/accounts-client.go
       #
-      # - mkdir -p mocks/grpc-interfaces/kloudlite.io/rpc/message-office-internal
+      # - mkdir -p mocks/grpc-interfaces/github.com/kloudlite/api/rpc/message-office-internal
       #
       # # pkg/kafka mocks
       # - mkdir -p mocks/pkg/kafka
-      # - mocki --package kloudlite.io/pkg/kafka --interface Producer > mocks/pkg/kafka/producer.go
-      # - mocki --package kloudlite.io/pkg/kafka --interface Consumer > mocks/pkg/kafka/consumer.go
+      # - mocki --package github.com/kloudlite/api/pkg/kafka --interface Producer > mocks/pkg/kafka/producer.go
+      # - mocki --package github.com/kloudlite/api/pkg/kafka --interface Consumer > mocks/pkg/kafka/consumer.go
       #

apps/container-registry/internal/domain/build.go

@@ -245,7 +245,8 @@ func (d *Impl) TriggerBuild(ctx RegistryContext, buildId repos.ID) error {
 		return errors.Newf("provider %s not supported", b.Source.Provider)
 	}
 
-	at, err := d.getAccessTokenByUserId(ctx, string(b.Source.Provider), ctx.UserId)
+	// at, err := d.getAccessTokenByUserId(ctx, string(b.Source.Provider), ctx.UserId)
+	at, err := d.getAccessTokenByUserId(ctx, string(b.Source.Provider), b.CreatedBy.UserId)
 	if err != nil {
 		return errors.NewE(err)
 	}

apps/infra/internal/domain/clusters.go

@@ -1,6 +1,7 @@
 package domain
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 
@@ -318,31 +319,45 @@ func (d *domain) CreateCluster(ctx InfraContext, cluster entities.Cluster) (*ent
 
 	d.resourceEventPublisher.PublishInfraEvent(ctx, ResourceTypeCluster, nCluster.Name, PublishAdd)
 
-	if err := d.applyHelmKloudliteAgent(ctx, nCluster.Name, string(tokenScrt.Data[keyClusterToken]), nCluster.Spec.PublicDNSHost, string(nCluster.Spec.CloudProvider)); err != nil {
+	if err := d.applyHelmKloudliteAgent(ctx, string(tokenScrt.Data[keyClusterToken]), nCluster); err != nil {
 		return nil, errors.NewE(err)
 	}
 
 	return nCluster, nil
 }
 
-func (d *domain) applyHelmKloudliteAgent(ctx InfraContext, clusterName string, clusterToken string, clusterPublicHost string, cloudprovider string) error {
+func (d *domain) applyHelmKloudliteAgent(ctx InfraContext, clusterToken string, cluster *entities.Cluster) error {
 	b, err := templates.Read(templates.HelmKloudliteAgent)
 	if err != nil {
 		return errors.NewE(err)
 	}
 
-	b2, err := templates.ParseBytes(b, map[string]any{
+	values := map[string]any{
 		"account-name": ctx.AccountName,
 
-		"cluster-name":  clusterName,
+		"cluster-name":  cluster.Name,
 		"cluster-token": clusterToken,
 
 		"kloudlite-release":        d.env.KloudliteRelease,
 		"message-office-grpc-addr": d.env.MessageOfficeExternalGrpcAddr,
 
-		"public-dns-host": clusterPublicHost,
-		"cloudprovider":   cloudprovider,
-	})
+		"public-dns-host": cluster.Spec.PublicDNSHost,
+		"cloudprovider":   cluster.Spec.CloudProvider,
+	}
+
+	if cluster.Spec.CloudProvider == ct.CloudProviderGCP {
+		var credsSecret corev1.Secret
+		if err := d.k8sClient.Get(ctx, fn.NN(cluster.Spec.GCP.CredentialsRef.Namespace, cluster.Spec.GCP.CredentialsRef.Name), &credsSecret); err != nil {
+			return err
+		}
+		gcpCreds, err := fn.JsonConvert[clustersv1.GCPCredentials](credsSecret.Data)
+		if err != nil {
+			return err
+		}
+		values["gcp-service-account-json"] = base64.StdEncoding.EncodeToString([]byte(gcpCreds.ServiceAccountJSON))
+	}
+
+	b2, err := templates.ParseBytes(b, values)
 	if err != nil {
 		return errors.NewE(err)
 	}
@@ -373,18 +388,18 @@ func (d *domain) applyHelmKloudliteAgent(ctx InfraContext, clusterName string, c
 			},
 		},
 		AccountName: ctx.AccountName,
-		ClusterName: clusterName,
+		ClusterName: cluster.Name,
 		SyncStatus:  t.GenSyncStatus(t.SyncActionApply, 0),
 	}
 
 	hr.IncrementRecordVersion()
 
-	uhr, err := d.upsertHelmRelease(ctx, clusterName, &hr)
+	uhr, err := d.upsertHelmRelease(ctx, cluster.Name, &hr)
 	if err != nil {
 		return errors.NewE(err)
 	}
 
-	if err := d.resDispatcher.ApplyToTargetCluster(ctx, clusterName, &uhr.HelmChart, uhr.RecordVersion); err != nil {
+	if err := d.resDispatcher.ApplyToTargetCluster(ctx, cluster.Name, &uhr.HelmChart, uhr.RecordVersion); err != nil {
 		return errors.NewE(err)
 	}
 
@@ -405,7 +420,7 @@ func (d *domain) UpgradeHelmKloudliteAgent(ctx InfraContext, clusterName string)
 		return errors.NewE(err)
 	}
 
-	if err := d.applyHelmKloudliteAgent(ctx, clusterName, out.ClusterToken, cluster.Spec.PublicDNSHost, string(cluster.Spec.CloudProvider)); err != nil {
+	if err := d.applyHelmKloudliteAgent(ctx, out.ClusterToken, cluster); err != nil {
 		return errors.NewE(err)
 	}
 

apps/infra/internal/domain/clusters_test

@@ -42,7 +42,6 @@ func TestCreateCluster(t *testing.T) {
 
 		k8sClient k8sMock.Client
 
-
 		iamClient                   iamMock.IAMClient
 		accountsSvc                 domainMocks.AccountsSvc
 		messageOfficeInternalClient msgOfficeInternalMock.MessageOfficeInternalClient

apps/infra/internal/domain/templates/helm-charts-kloudlite-agent.yml.tpl

@@ -9,6 +9,12 @@
 
 {{- $cloudprovider := get . "cloudprovider" }}
 
+{{- $gcpServiceAccountJson := get . "gcp-service-account-json" | default "" }}
+
+{{- if and (eq $cloudprovider "gcp") (not $gcpServiceAccountJson) }}
+{{fail "when cloudprovider is gcp, field `gcp-service-account-json` is required"}}
+{{- end }}
+
 ---
 apiVersion: crds.kloudlite.io/v1
 kind: HelmChart
@@ -25,7 +31,7 @@ spec:
       - operator: Exists
 
   preInstall: |+
-    kubectl apply -f https://github.com/kloudlite/helm-charts/releases/download/{{$kloudliteRelease}}/crds-kloudlite.yml
+    kubectl apply -f https://github.com/kloudlite/helm-charts/releases/download/{{$kloudliteRelease}}/crds-kloudlite.yml --server-side
 
   postInstall: |+
     if kubectl get ns kloudlite-tmp;
@@ -58,6 +64,13 @@ spec:
 
     cloudProvider: "{{$cloudprovider}}"
 
+    {{- if (eq $cloudprovider "gcp") }}
+    gcp:
+      gcloudServiceAccountCreds:
+        # -- base64 encoded service account json
+        json: {{$gcpServiceAccountJson}}
+    {{- end }}
+
     agent:
       enabled: true
       name: kl-agent

apps/infra/internal/entities/field-constants/generated_constants.go

@@ -75,6 +75,8 @@ const (
 	BYOKClusterGcpMasterNodesRootVolumeSize        = "gcp.masterNodes.rootVolumeSize"
 	BYOKClusterGcpMasterNodesRootVolumeType        = "gcp.masterNodes.rootVolumeType"
 	BYOKClusterGcpRegion                           = "gcp.region"
+	BYOKClusterGcpVpc                              = "gcp.vpc"
+	BYOKClusterGcpVpcName                          = "gcp.vpc.name"
 	BYOKClusterKloudliteRelease                    = "kloudliteRelease"
 	BYOKClusterMessageQueueTopicName               = "messageQueueTopicName"
 	BYOKClusterOutput                              = "output"
@@ -155,6 +157,8 @@ const (
 	ClusterSpecGcpMasterNodesRootVolumeSize        = "spec.gcp.masterNodes.rootVolumeSize"
 	ClusterSpecGcpMasterNodesRootVolumeType        = "spec.gcp.masterNodes.rootVolumeType"
 	ClusterSpecGcpRegion                           = "spec.gcp.region"
+	ClusterSpecGcpVpc                              = "spec.gcp.vpc"
+	ClusterSpecGcpVpcName                          = "spec.gcp.vpc.name"
 	ClusterSpecKloudliteRelease                    = "spec.kloudliteRelease"
 	ClusterSpecMessageQueueTopicName               = "spec.messageQueueTopicName"
 	ClusterSpecOutput                              = "spec.output"
@@ -172,7 +176,9 @@ const (
 
 // constant vars generated for struct ClusterConnection
 const (
+	ClusterConnectionCidr                         = "cidr"
 	ClusterConnectionClusterGroupName             = "clusterGroupName"
+	ClusterConnectionEndpoint                     = "endpoint"
 	ClusterConnectionSpec                         = "spec"
 	ClusterConnectionSpecAgentsResources          = "spec.agentsResources"
 	ClusterConnectionSpecAgentsResourcesClaims    = "spec.agentsResources.claims"
@@ -192,9 +198,7 @@ const (
 )
 
 // constant vars generated for struct ClusterGroup
-const (
-	ClusterGroupPeers = "peers"
-)
+const ()
 
 // constant vars generated for struct ClusterManagedService
 const (
@@ -378,12 +382,10 @@ const (
 	OutputFieldName        = "name"
 )
 
-// constant vars generated for struct Peers
+// constant vars generated for struct Peer
 const (
-	PeersAllowedIps = "allowedIps"
-	PeersName       = "name"
-	PeersNamespace  = "namespace"
-	PeersPubKey     = "pubKey"
+	PeerAllowedIps = "allowedIps"
+	PeerPubKey     = "pubKey"
 )
 
 // constant vars generated for struct PersistentVolume

cmd/mocki/Taskfile.yml

@@ -3,6 +3,8 @@ version: 3
 tasks:
   install:
     dir: ../..
+    env:
+      CGO_ENABLED: 0
     cmds: 
       - mkdir -p bin
       - go build -o ./bin/mocki ./cmd/mocki