diff --git a/pkg/utils/common.go b/pkg/utils/common.go index 73a07f62c..2a8d915c5 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -242,10 +242,14 @@ func ShouldPropagateObj(informerManager informer.Manager, uObj *unstructured.Uns return true, nil } -// ShouldPropagateNamespace decides if we should propagate the resources in the namespace +// IsReservedNamespace indicates if an argued namespace is reserved. +func IsReservedNamespace(namespace string) bool { + return strings.HasPrefix(namespace, fleetPrefix) || strings.HasPrefix(namespace, kubePrefix) +} + +// ShouldPropagateNamespace decides if we should propagate the resources in the namespace. func ShouldPropagateNamespace(namespace string, skippedNamespaces map[string]bool) bool { - // special case for namespace have the reserved prefix - if strings.HasPrefix(namespace, fleetPrefix) || strings.HasPrefix(namespace, kubePrefix) { + if IsReservedNamespace(namespace) { return false } diff --git a/pkg/webhook/add_replicaset.go b/pkg/webhook/add_replicaset.go new file mode 100644 index 000000000..9d1e37e90 --- /dev/null +++ b/pkg/webhook/add_replicaset.go @@ -0,0 +1,14 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package webhook + +import ( + "go.goms.io/fleet/pkg/webhook/replicaset" +) + +func init() { + AddToManagerFuncs = append(AddToManagerFuncs, replicaset.Add) +} diff --git a/pkg/webhook/pod/pod_validating_webhook.go b/pkg/webhook/pod/pod_validating_webhook.go index a16d491f1..b878bc43f 100644 --- a/pkg/webhook/pod/pod_validating_webhook.go +++ b/pkg/webhook/pod/pod_validating_webhook.go @@ -16,6 +16,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "go.goms.io/fleet/pkg/utils" ) const ( @@ -43,7 +45,7 @@ func (v *podValidator) Handle(ctx context.Context, req admission.Request) admiss if err != nil { return admission.Errored(http.StatusBadRequest, err) } - if pod.Namespace != "kube-system" && pod.Namespace != "fleet-system" { + if !utils.IsReservedNamespace(pod.Namespace) { return admission.Denied(fmt.Sprintf("Pod %s/%s creation is disallowed in the fleet hub cluster", pod.Namespace, pod.Name)) } } diff --git a/pkg/webhook/replicaset/replicaset_validating_webhook.go b/pkg/webhook/replicaset/replicaset_validating_webhook.go new file mode 100644 index 000000000..f94f4ab90 --- /dev/null +++ b/pkg/webhook/replicaset/replicaset_validating_webhook.go @@ -0,0 +1,58 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package replicaset + +import ( + "context" + "fmt" + "net/http" + + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "go.goms.io/fleet/pkg/utils" +) + +const ( + // ValidationPath is the webhook service path which admission requests are routed to for validating ReplicaSet resources. + ValidationPath = "/validate-apps-v1-replicaset" +) + +type replicaSetValidator struct { + Client client.Client + decoder *admission.Decoder +} + +// Add registers the webhook for K8s bulit-in object types. +func Add(mgr manager.Manager) error { + hookServer := mgr.GetWebhookServer() + hookServer.Register(ValidationPath, &webhook.Admission{Handler: &replicaSetValidator{Client: mgr.GetClient()}}) + return nil +} + +// Handle replicaSetValidator denies all creation requests. +func (v *replicaSetValidator) Handle(ctx context.Context, req admission.Request) admission.Response { + if req.Operation == admissionv1.Create { + rs := &v1.ReplicaSet{} + if err := v.decoder.Decode(req, rs); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + if !utils.IsReservedNamespace(rs.Namespace) { + return admission.Denied(fmt.Sprintf("ReplicaSet %s/%s creation is disallowed in the fleet hub cluster.", rs.Namespace, rs.Name)) + } + } + return admission.Allowed("") +} + +// InjectDecoder injects the decoder. +func (v *replicaSetValidator) InjectDecoder(d *admission.Decoder) error { + v.decoder = d + return nil +} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 9868f86f1..e0d7198a8 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -21,6 +21,7 @@ import ( "time" admv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,6 +33,7 @@ import ( "go.goms.io/fleet/cmd/hubagent/options" "go.goms.io/fleet/pkg/webhook/clusterresourceplacement" "go.goms.io/fleet/pkg/webhook/pod" + "go.goms.io/fleet/pkg/webhook/replicaset" ) const ( @@ -82,7 +84,7 @@ func NewWebhookConfig(mgr manager.Manager, port int, clientConnectionType *optio } caPEM, err := w.genCertificate(certDir) if err != nil { - return nil, err // TODO + return nil, err } w.caPEM = caPEM return &w, err @@ -97,9 +99,9 @@ func (w *Config) Start(ctx context.Context) error { return nil } -// createFleetWebhookConfiguration creates the ValidatingWebhookConfiguration object for the webhook +// createFleetWebhookConfiguration creates the ValidatingWebhookConfiguration object for the webhook. func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error { - failPolicy := admv1.Fail // reject request if the webhook doesn't work + failPolicy := admv1.Fail sideEffortsNone := admv1.SideEffectClassNone namespacedScope := admv1.NamespacedScope clusterScope := admv1.ClusterScope @@ -155,6 +157,26 @@ func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error { }, }, }, + { + Name: "fleet.replicaset.validating", + ClientConfig: w.createClientConfig(appsv1.ReplicaSet{}), + FailurePolicy: &failPolicy, + SideEffects: &sideEffortsNone, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, + Rules: []admv1.RuleWithOperations{ + { + Operations: []admv1.OperationType{ + admv1.Create, + }, + Rule: admv1.Rule{ + APIGroups: []string{"apps"}, + APIVersions: []string{"v1"}, + Resources: []string{"replicasets"}, + Scope: &namespacedScope, + }, + }, + }, + }, }, } @@ -178,6 +200,7 @@ func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error { return nil } +// createClientConfig generates the client configuration with either service ref or URL for the argued interface. func (w *Config) createClientConfig(webhookInterface interface{}) admv1.WebhookClientConfig { serviceRef := admv1.ServiceReference{ Namespace: w.serviceNamespace, @@ -192,6 +215,9 @@ func (w *Config) createClientConfig(webhookInterface interface{}) admv1.WebhookC case fleetv1alpha1.ClusterResourcePlacement: serviceEndpoint = w.serviceURL + clusterresourceplacement.ValidationPath serviceRef.Path = pointer.String(clusterresourceplacement.ValidationPath) + case appsv1.ReplicaSet: + serviceEndpoint = w.serviceURL + replicaset.ValidationPath + serviceRef.Path = pointer.String(replicaset.ValidationPath) } config := admv1.WebhookClientConfig{ @@ -206,7 +232,7 @@ func (w *Config) createClientConfig(webhookInterface interface{}) admv1.WebhookC return config } -// genCertificate generates the serving cerficiate for the webhook server +// genCertificate generates the serving cerficiate for the webhook server. func (w *Config) genCertificate(certDir string) ([]byte, error) { caPEM, certPEM, keyPEM, err := w.genSelfSignedCert() if err != nil { diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index a723f01a1..e66757bf5 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -43,17 +43,31 @@ var ( imc *v1alpha1.InternalMemberCluster ctx context.Context - // This namespace will store Member cluster-related CRs, such as v1alpha1.MemberCluster + // The fleet-system namespace. + fleetSystemNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fleet-system", + }, + } + + // The kube-system namespace + kubeSystemNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-system", + }, + } + + // This namespace will store Member cluster-related CRs, such as v1alpha1.MemberCluster. memberNamespace = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName), + Name: fmt.Sprintf("fleet-member-%s", MemberCluster.ClusterName), }, } // This namespace in HubCluster will store v1alpha1.Work to simulate Work-related features in Hub Cluster. workNamespace = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName), + Name: fmt.Sprintf("fleet-member-%s", MemberCluster.ClusterName), }, } @@ -174,7 +188,7 @@ var _ = BeforeSuite(func() { identity := rbacv1.Subject{ Name: "member-agent-sa", Kind: "ServiceAccount", - Namespace: "fleet-system", + Namespace: utils.FleetSystemNamespace, } mc = &v1alpha1.MemberCluster{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/webhook_test.go b/test/e2e/webhook_test.go index dcead2c36..db771032c 100644 --- a/test/e2e/webhook_test.go +++ b/test/e2e/webhook_test.go @@ -13,9 +13,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1" @@ -23,23 +25,19 @@ import ( testUtils "go.goms.io/fleet/test/e2e/utils" ) -const ( - kubeSystemNs = "kube-system" - fleetSystemNs = "fleet-system" -) - var ( - whitelistedNamespaces = []corev1.Namespace{ - {ObjectMeta: metav1.ObjectMeta{Name: kubeSystemNs}}, - {ObjectMeta: metav1.ObjectMeta{Name: fleetSystemNs}}, + reservedNamespaces = []*corev1.Namespace{ + fleetSystemNamespace, + kubeSystemNamespace, + memberNamespace, } ) var _ = Describe("Fleet's Hub cluster webhook tests", func() { Context("Pod validation webhook", func() { - It("should admit operations on Pods within system reserved namespaces", func() { - for _, ns := range whitelistedNamespaces { - objKey := client.ObjectKey{Name: utils.RandStr(), Namespace: ns.ObjectMeta.Name} + It("should admit operations on Pods within reserved namespaces", func() { + for _, reservedNamespace := range reservedNamespaces { + objKey := client.ObjectKey{Name: utils.RandStr(), Namespace: reservedNamespace.Name} nginxPod := &corev1.Pod{ TypeMeta: metav1.TypeMeta{ Kind: "Pod", @@ -66,10 +64,10 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { }, } - By(fmt.Sprintf("expecting admission of operation CREATE in system reserved namespace %s", ns.ObjectMeta.Name)) + By(fmt.Sprintf("expecting admission of operation CREATE of Pod in reserved namespace %s", reservedNamespace.Name)) Expect(HubCluster.KubeClient.Create(ctx, nginxPod)).Should(Succeed()) - By(fmt.Sprintf("expecting admission of operation UPDATE in system reserved namespace %s", ns.ObjectMeta.Name)) + By(fmt.Sprintf("expecting admission of operation UPDATE of Pod in reserved namespace %s", reservedNamespace.Name)) var podV2 *corev1.Pod Eventually(func() error { var currentPod corev1.Pod @@ -79,11 +77,11 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { return HubCluster.KubeClient.Update(ctx, podV2) }, testUtils.PollTimeout, testUtils.PollInterval).Should(Succeed()) - By(fmt.Sprintf("expecting admission of operation DELETE in system reserved namespace %s", ns.ObjectMeta.Name)) + By(fmt.Sprintf("expecting admission of operation DELETE of Pod in reserved namespace %s", reservedNamespace.Name)) Expect(HubCluster.KubeClient.Delete(ctx, nginxPod)).Should(Succeed()) } }) - It("should deny create operation on Pods within any non-system reserved namespace", func() { + It("should deny create operation on Pods within any non-reserved namespace", func() { rndNs := corev1.Namespace{ TypeMeta: metav1.TypeMeta{ Kind: "namespace", @@ -121,7 +119,7 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { }, } - By(fmt.Sprintf("expecting denial of operation CREATE in non-system reserved namespace %s", rndNs.Name)) + By(fmt.Sprintf("expecting denial of operation CREATE of Pod in non-reserved namespace %s", rndNs.Name)) err := HubCluster.KubeClient.Create(ctx, nginxPod) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create Pod call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) @@ -150,7 +148,7 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { }, }, } - By("expecting admission of operation CREATE of a valid CRP") + By("expecting admission of operation CREATE of a valid ClusterResourcePlacement") Expect(HubCluster.KubeClient.Create(ctx, &validCRP)).Should(Succeed()) // Get the created CRP @@ -166,12 +164,12 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { }, testUtils.PollTimeout, testUtils.PollInterval).Should(Succeed()) }) AfterEach(func() { - By("expecting admission of operation DELETE") + By("expecting admission of operation DELETE of ClusterResourcePlacement") Expect(HubCluster.KubeClient.Delete(ctx, &createdCRP)).Should(Succeed()) }) It("should admit write operations for valid ClusterResourcePlacement resources", func() { // create & delete write operations are handled within the BeforeEach & AfterEach functions. - By("expecting admission of operation UPDATE with a valid CRP") + By("expecting admission of operation UPDATE with a valid ClusterResourcePlacement") createdCRP.Spec.ResourceSelectors[0].Name = utils.RandStr() Expect(HubCluster.KubeClient.Update(ctx, &createdCRP)).Should(Succeed()) }) @@ -199,7 +197,7 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { }, } - By("expecting denial of operation CREATE with the invalid CRP") + By("expecting denial of operation CREATE with the invalid ClusterResourcePlacement") err := HubCluster.KubeClient.Create(ctx, &invalidCRP) Expect(err).Should(HaveOccurred()) var statusErr *k8sErrors.StatusError @@ -207,7 +205,7 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("the labelSelector and name fields are mutually exclusive")) - By("expecting denial of operation UPDATE with the invalid CRP") + By("expecting denial of operation UPDATE with the invalid ClusterResourcePlacement") createdCRP.Spec = invalidCRP.Spec err = HubCluster.KubeClient.Update(ctx, &createdCRP) Expect(err).Should(HaveOccurred()) @@ -250,14 +248,14 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { }, } - By("expecting denial of operation CREATE") + By("expecting denial of operation CREATE of ClusterResourcePlacement") err := HubCluster.KubeClient.Create(ctx, &invalidCRP) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(regexp.QuoteMeta(fmt.Sprintf("the labelSelector in cluster selector %+v is invalid:", invalidCRP.Spec.Policy.Affinity.ClusterAffinity.ClusterSelectorTerms[0])))) - By("expecting denial of operation UPDATE") + By("expecting denial of operation UPDATE of ClusterResourcePlacement") createdCRP.Spec = invalidCRP.Spec err = HubCluster.KubeClient.Update(ctx, &createdCRP) Expect(err).Should(HaveOccurred()) @@ -296,14 +294,14 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { }, } - By("expecting denial of operation CREATE with the invalid CRP") + By("expecting denial of operation CREATE with the invalid ClusterResourcePlacement") err := HubCluster.KubeClient.Create(ctx, &invalidCRP) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(regexp.QuoteMeta(fmt.Sprintf("the labelSelector in resource selector %+v is invalid:", invalidCRP.Spec.ResourceSelectors[0])))) - By("expecting denial of operation UPDATE with the invalid CRP") + By("expecting denial of operation UPDATE with the invalid ClusterResourcePlacement") createdCRP.Spec = invalidCRP.Spec err = HubCluster.KubeClient.Update(ctx, &createdCRP) Expect(err).Should(HaveOccurred()) @@ -339,14 +337,14 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { Kind: invalidCRP.Spec.ResourceSelectors[0].Kind, } - By("expecting denial of operation CREATE with the invalid CRP") + By("expecting denial of operation CREATE with the invalid ClusterResourcePlacement") err := HubCluster.KubeClient.Create(ctx, &invalidCRP) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterResourcePlacement call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.clusterresourceplacement.validating" denied the request`)) Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(regexp.QuoteMeta(fmt.Sprintf("the resource is not found in schema (please retry) or it is not a cluster scoped resource: %s", invalidGVK)))) - By("expecting denial of operation UPDATE with the invalid CRP") + By("expecting denial of operation UPDATE with the invalid ClusterResourcePlacement") createdCRP.Spec = invalidCRP.Spec err = HubCluster.KubeClient.Update(ctx, &createdCRP) Expect(err).Should(HaveOccurred()) @@ -355,4 +353,110 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(regexp.QuoteMeta(fmt.Sprintf("the resource is not found in schema (please retry) or it is not a cluster scoped resource: %s", invalidGVK)))) }) }) + Context("ReplicaSet validation webhook", func() { + It("should admit operation CREATE on ReplicaSets in reserved namespaces", func() { + for _, ns := range reservedNamespaces { + rs := &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicaSet", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + Namespace: ns.Name, + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: pointer.Int32(1), + MinReadySeconds: 1, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "app", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"web"}, + }, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "web"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + Ports: []corev1.ContainerPort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + } + + By(fmt.Sprintf("expecting admission of operation CREATE of ReplicaSet in reserved namespace %s", ns.Name)) + Expect(HubCluster.KubeClient.Create(ctx, rs)).Should(Succeed()) + } + }) + It("should deny CREATE operation on ReplicaSets in a non-reserved namespace", func() { + rs := &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicaSet", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RandStr(), + Namespace: "default", + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: pointer.Int32(1), + MinReadySeconds: 1, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "app", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"web"}, + }, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "web"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + Ports: []corev1.ContainerPort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + } + + By("expecting denial of operation CREATE of ReplicaSet") + err := HubCluster.KubeClient.Create(ctx, rs) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ReplicaSet call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(`admission webhook "fleet.replicaset.validating" denied the request`)) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp(fmt.Sprintf("ReplicaSet %s/%s creation is disallowed in the fleet hub cluster", rs.Namespace, rs.Name))) + }) + }) })