diff --git a/pkg/utils/test_util.go b/pkg/utils/test_util.go index 4adb45093..2b9c8798e 100644 --- a/pkg/utils/test_util.go +++ b/pkg/utils/test_util.go @@ -10,16 +10,15 @@ import ( "os" "strings" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/client-go/kubernetes/scheme" - "github.com/onsi/gomega/format" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" ) @@ -63,7 +62,6 @@ func GetObjectFromManifest(relativeFilePath string, obj runtime.Object) error { if err != nil { return err } - return GetObjectFromRawExtension(fileRaw, obj) } diff --git a/pkg/webhook/add_fleetresourcehandler.go b/pkg/webhook/add_fleetresourcehandler.go new file mode 100644 index 000000000..3172bcc19 --- /dev/null +++ b/pkg/webhook/add_fleetresourcehandler.go @@ -0,0 +1,8 @@ +package webhook + +import "go.goms.io/fleet/pkg/webhook/fleetresourcehandler" + +func init() { + // AddToManagerFuncs is a list of functions to create webhook and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, fleetresourcehandler.Add) +} diff --git a/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook.go b/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook.go new file mode 100644 index 000000000..99640070c --- /dev/null +++ b/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook.go @@ -0,0 +1,89 @@ +package fleetresourcehandler + +import ( + "context" + "fmt" + "net/http" + "regexp" + + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "go.goms.io/fleet/pkg/webhook/validation" +) + +const ( + // ValidationPath is the webhook service path which admission requests are routed to for validating custom resource definition resources. + ValidationPath = "/validate-v1-fleetresourcehandler" + groupMatch = `^[^.]*\.(.*)` + crdKind = "CustomResourceDefinition" +) + +// Add registers the webhook for K8s bulit-in object types. +func Add(mgr manager.Manager) error { + hookServer := mgr.GetWebhookServer() + hookServer.Register(ValidationPath, &webhook.Admission{Handler: &fleetResourceValidator{Client: mgr.GetClient()}}) + return nil +} + +type fleetResourceValidator struct { + Client client.Client + decoder *admission.Decoder +} + +func (v *fleetResourceValidator) Handle(_ context.Context, req admission.Request) admission.Response { + var response admission.Response + if req.Operation == admissionv1.Create || req.Operation == admissionv1.Update || req.Operation == admissionv1.Delete { + switch req.Kind { + case createCRDGVK(): + klog.V(2).InfoS("handling CRD resource", "GVK", createCRDGVK()) + response = v.handleCRD(req) + default: + klog.V(2).InfoS("resource is not monitored by fleet resource validator webhook", "GVK", req.Kind.String()) + response = admission.Allowed(fmt.Sprintf("user: %s in groups: %v is allowed to modify resource with GVK: %s", req.UserInfo.Username, req.UserInfo.Groups, req.Kind.String())) + } + } + return response +} + +func (v *fleetResourceValidator) handleCRD(req admission.Request) admission.Response { + var crd v1.CustomResourceDefinition + if req.Operation == admissionv1.Delete { + // req.Object is not populated for delete: https://github.com/kubernetes-sigs/controller-runtime/issues/1762. + if err := v.decoder.DecodeRaw(req.OldObject, &crd); err != nil { + klog.ErrorS(err, "failed to decode old request object for delete operation", "userName", req.UserInfo.Username, "groups", req.UserInfo.Groups) + return admission.Errored(http.StatusBadRequest, err) + } + } else { + if err := v.decoder.Decode(req, &crd); err != nil { + klog.ErrorS(err, "failed to decode request object for create/update operation", "userName", req.UserInfo.Username, "groups", req.UserInfo.Groups) + return admission.Errored(http.StatusBadRequest, err) + } + } + + // This regex works because every CRD name in kubernetes follows this pattern .. + group := regexp.MustCompile(groupMatch).FindStringSubmatch(crd.Name)[1] + if validation.CheckCRDGroup(group) && !validation.ValidateUserForCRD(req.UserInfo) { + return admission.Denied(fmt.Sprintf("failed to validate user: %s in groups: %v to modify fleet CRD: %s", req.UserInfo.Username, req.UserInfo.Groups, crd.Name)) + } + return admission.Allowed(fmt.Sprintf("user: %s in groups: %v is allowed to modify CRD: %s", req.UserInfo.Username, req.UserInfo.Groups, crd.Name)) +} + +func createCRDGVK() metav1.GroupVersionKind { + return metav1.GroupVersionKind{ + Group: v1.SchemeGroupVersion.Group, + Version: v1.SchemeGroupVersion.Version, + Kind: crdKind, + } +} + +func (v *fleetResourceValidator) InjectDecoder(d *admission.Decoder) error { + v.decoder = d + return nil +} diff --git a/pkg/webhook/validation/crdvalidation.go b/pkg/webhook/validation/crdvalidation.go new file mode 100644 index 000000000..40826634c --- /dev/null +++ b/pkg/webhook/validation/crdvalidation.go @@ -0,0 +1,12 @@ +package validation + +import "k8s.io/utils/strings/slices" + +var ( + validObjectGroups = []string{"networking.fleet.azure.com", "fleet.azure.com", "multicluster.x-k8s.io", "placement.karavel.io"} +) + +// CheckCRDGroup checks to see if the input CRD group is a fleet CRD group. +func CheckCRDGroup(group string) bool { + return slices.Contains(validObjectGroups, group) +} diff --git a/pkg/webhook/validation/uservalidation.go b/pkg/webhook/validation/uservalidation.go new file mode 100644 index 000000000..cc4e7ec01 --- /dev/null +++ b/pkg/webhook/validation/uservalidation.go @@ -0,0 +1,17 @@ +package validation + +import ( + authenticationv1 "k8s.io/api/authentication/v1" + "k8s.io/utils/strings/slices" +) + +const ( + mastersGroup = "system:masters" +) + +// TODO:(Arvindthiru) Get valid usernames as flag and allow those usernames. + +// ValidateUserForCRD checks to see if user is authenticated to make a request to modify fleet CRDs. +func ValidateUserForCRD(userInfo authenticationv1.UserInfo) bool { + return slices.Contains(userInfo.Groups, mastersGroup) +} diff --git a/pkg/webhook/validation/uservalidation_test.go b/pkg/webhook/validation/uservalidation_test.go new file mode 100644 index 000000000..d21d1a883 --- /dev/null +++ b/pkg/webhook/validation/uservalidation_test.go @@ -0,0 +1,53 @@ +package validation + +import ( + "context" + "testing" + + "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/authentication/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.goms.io/fleet/pkg/utils" +) + +func TestValidateUserForCRD(t *testing.T) { + testCases := map[string]struct { + client client.Client + userInfo v1.UserInfo + wantResult bool + }{ + "allow user in system:masters group": { + client: &test.MockClient{ + MockList: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + return nil + }, + }, + userInfo: v1.UserInfo{ + Username: "test-user", + Groups: []string{"system:masters"}, + }, + wantResult: true, + }, + "fail to validate user with invalid username, groups": { + client: &test.MockClient{ + MockList: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + return nil + }, + }, + userInfo: v1.UserInfo{ + Username: "test-user", + Groups: []string{"test-group"}, + }, + wantResult: false, + }, + } + + for testName, testCase := range testCases { + t.Run(testName, func(t *testing.T) { + gotResult := ValidateUserForCRD(testCase.userInfo) + assert.Equal(t, testCase.wantResult, gotResult, utils.TestCaseMsg, testName) + }) + } +} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index ea38fa539..97e151ca1 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -21,8 +21,10 @@ import ( "time" admv1 "k8s.io/api/admissionregistration/v1" + admv1beta1 "k8s.io/api/admissionregistration/v1beta1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" @@ -33,6 +35,7 @@ import ( fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/cmd/hubagent/options" "go.goms.io/fleet/pkg/webhook/clusterresourceplacement" + "go.goms.io/fleet/pkg/webhook/fleetresourcehandler" "go.goms.io/fleet/pkg/webhook/pod" "go.goms.io/fleet/pkg/webhook/replicaset" ) @@ -42,6 +45,14 @@ const ( FleetWebhookKeyFileName = "tls.key" FleetWebhookCfgName = "fleet-validating-webhook-configuration" FleetWebhookSvcName = "fleetwebhook" + + crdResourceName = "customresourcedefinitions" + replicaSetResourceName = "replicasets" + podResourceName = "pods" +) + +var ( + admissionReviewVersions = []string{admv1.SchemeGroupVersion.Version, admv1beta1.SchemeGroupVersion.Version} ) var AddToManagerFuncs []func(manager.Manager) error @@ -120,7 +131,7 @@ func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error { ClientConfig: w.createClientConfig(corev1.Pod{}), FailurePolicy: &failPolicy, SideEffects: &sideEffortsNone, - AdmissionReviewVersions: []string{"v1", "v1beta1"}, + AdmissionReviewVersions: admissionReviewVersions, Rules: []admv1.RuleWithOperations{ { @@ -128,9 +139,9 @@ func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error { admv1.Create, }, Rule: admv1.Rule{ - APIGroups: []string{""}, - APIVersions: []string{"v1"}, - Resources: []string{"pods"}, + APIGroups: []string{corev1.SchemeGroupVersion.Group}, + APIVersions: []string{corev1.SchemeGroupVersion.Version}, + Resources: []string{podResourceName}, Scope: &namespacedScope, }, }, @@ -141,7 +152,7 @@ func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error { ClientConfig: w.createClientConfig(fleetv1alpha1.ClusterResourcePlacement{}), FailurePolicy: &failPolicy, SideEffects: &sideEffortsNone, - AdmissionReviewVersions: []string{"v1", "v1beta1"}, + AdmissionReviewVersions: admissionReviewVersions, Rules: []admv1.RuleWithOperations{ { @@ -150,8 +161,8 @@ func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error { admv1.Update, }, Rule: admv1.Rule{ - APIGroups: []string{"fleet.azure.com"}, - APIVersions: []string{"v1alpha1"}, + APIGroups: []string{fleetv1alpha1.GroupVersion.Group}, + APIVersions: []string{fleetv1alpha1.GroupVersion.Version}, Resources: []string{fleetv1alpha1.ClusterResourcePlacementResource}, Scope: &clusterScope, }, @@ -163,21 +174,43 @@ func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error { ClientConfig: w.createClientConfig(appsv1.ReplicaSet{}), FailurePolicy: &failPolicy, SideEffects: &sideEffortsNone, - AdmissionReviewVersions: []string{"v1", "v1beta1"}, + AdmissionReviewVersions: admissionReviewVersions, Rules: []admv1.RuleWithOperations{ { Operations: []admv1.OperationType{ admv1.Create, }, Rule: admv1.Rule{ - APIGroups: []string{"apps"}, - APIVersions: []string{"v1"}, - Resources: []string{"replicasets"}, + APIGroups: []string{appsv1.SchemeGroupVersion.Group}, + APIVersions: []string{appsv1.SchemeGroupVersion.Version}, + Resources: []string{replicaSetResourceName}, Scope: &namespacedScope, }, }, }, }, + { + Name: "fleet.customresourcedefinition.validating", + ClientConfig: w.createClientConfig(v1.CustomResourceDefinition{}), + FailurePolicy: &failPolicy, + SideEffects: &sideEffortsNone, + AdmissionReviewVersions: admissionReviewVersions, + Rules: []admv1.RuleWithOperations{ + { + Operations: []admv1.OperationType{ + admv1.Create, + admv1.Update, + admv1.Delete, + }, + Rule: admv1.Rule{ + APIGroups: []string{v1.SchemeGroupVersion.Group}, + APIVersions: []string{v1.SchemeGroupVersion.Version}, + Resources: []string{crdResourceName}, + Scope: &clusterScope, + }, + }, + }, + }, }, } @@ -225,6 +258,9 @@ func (w *Config) createClientConfig(webhookInterface interface{}) admv1.WebhookC case appsv1.ReplicaSet: serviceEndpoint = w.serviceURL + replicaset.ValidationPath serviceRef.Path = pointer.String(replicaset.ValidationPath) + case v1.CustomResourceDefinition: + serviceEndpoint = w.serviceURL + fleetresourcehandler.ValidationPath + serviceRef.Path = pointer.String(fleetresourcehandler.ValidationPath) } config := admv1.WebhookClientConfig{ diff --git a/test/e2e/framework/cluster.go b/test/e2e/framework/cluster.go index b36156f3a..6f2e1f1a7 100644 --- a/test/e2e/framework/cluster.go +++ b/test/e2e/framework/cluster.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) @@ -22,12 +23,13 @@ var ( // Cluster object defines the required clients based on the kubeconfig of the test cluster. type Cluster struct { - Scheme *runtime.Scheme - KubeClient client.Client - DynamicClient dynamic.Interface - ClusterName string - HubURL string - RestMapper meta.RESTMapper + Scheme *runtime.Scheme + KubeClient client.Client + ImpersonateKubeClient client.Client + DynamicClient dynamic.Interface + ClusterName string + HubURL string + RestMapper meta.RESTMapper } func NewCluster(name string, scheme *runtime.Scheme) *Cluster { @@ -40,12 +42,18 @@ func NewCluster(name string, scheme *runtime.Scheme) *Cluster { // GetClusterClient returns a Cluster client for the cluster. func GetClusterClient(cluster *Cluster) { clusterConfig := GetClientConfig(cluster) + impersonateClusterConfig := GetImpersonateClientConfig(cluster) restConfig, err := clusterConfig.ClientConfig() if err != nil { gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up rest config") } + impersonateRestConfig, err := impersonateClusterConfig.ClientConfig() + if err != nil { + gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up impersonate rest config") + } + cluster.KubeClient, err = client.New(restConfig, client.Options{Scheme: cluster.Scheme}) gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up Kube Client") @@ -54,6 +62,9 @@ func GetClusterClient(cluster *Cluster) { cluster.RestMapper, err = apiutil.NewDynamicRESTMapper(restConfig, apiutil.WithLazyDiscovery) gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up Rest Mapper") + + cluster.ImpersonateKubeClient, err = client.New(impersonateRestConfig, client.Options{Scheme: cluster.Scheme}) + gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up Kube Client") } func GetClientConfig(cluster *Cluster) clientcmd.ClientConfig { @@ -63,3 +74,15 @@ func GetClientConfig(cluster *Cluster) clientcmd.ClientConfig { CurrentContext: cluster.ClusterName, }) } + +func GetImpersonateClientConfig(cluster *Cluster) clientcmd.ClientConfig { + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + &clientcmd.ConfigOverrides{ + CurrentContext: cluster.ClusterName, + AuthInfo: api.AuthInfo{ + Impersonate: "test-user", + ImpersonateGroups: []string{"system:authenticated"}, + }, + }) +} diff --git a/test/e2e/webhook_test.go b/test/e2e/webhook_test.go index db771032c..9f9046d7f 100644 --- a/test/e2e/webhook_test.go +++ b/test/e2e/webhook_test.go @@ -6,7 +6,7 @@ Licensed under the MIT license. package e2e import ( - errors "errors" + "errors" "fmt" "reflect" "regexp" @@ -15,8 +15,11 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,6 +34,17 @@ var ( kubeSystemNamespace, memberNamespace, } + testGroups = []string{"system:authenticated"} +) + +const ( + testUser = "test-user" + testKey = "test-key" + testValue = "test-value" + testUserClusterRole = "test-user-cluster-role" + testUserClusterRoleBinding = "test-user-cluster-role-binding" + + crdStatusErrFormat = `failed to validate user: %s in groups: %v to modify fleet CRD: %s` ) var _ = Describe("Fleet's Hub cluster webhook tests", func() { @@ -460,3 +474,132 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { }) }) }) + +var _ = Describe("Fleet's CRD Resource Handler webhook tests", func() { + BeforeEach(func() { + By("create cluster role to modify CRDs") + cr := rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: testUserClusterRole, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{v1.SchemeGroupVersion.Group}, + Verbs: []string{"*"}, + Resources: []string{"*"}, + }, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, &cr)).Should(Succeed()) + + By("create cluster role binding for test-user to modify CRD") + crb := rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testUserClusterRoleBinding, + }, + Subjects: []rbacv1.Subject{ + { + APIGroup: rbacv1.GroupName, + Kind: "User", + Name: testUser, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: testUserClusterRole, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, &crb)).Should(Succeed()) + }) + + AfterEach(func() { + By("remove cluster role binding") + crb := rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testUserClusterRoleBinding, + }, + } + Expect(HubCluster.KubeClient.Delete(ctx, &crb)).Should(Succeed()) + + By("remove cluster role") + cr := rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: testUserClusterRole, + }, + } + Expect(HubCluster.KubeClient.Delete(ctx, &cr)).Should(Succeed()) + }) + + Context("CRD validation webhook", func() { + It("should deny CREATE operation on Fleet CRD for user not in system:masters group", func() { + var crd v1.CustomResourceDefinition + Expect(utils.GetObjectFromManifest("./charts/hub-agent/templates/crds/fleet.azure.com_clusterresourceplacements.yaml", &crd)).Should(Succeed()) + + By("expecting denial of operation CREATE of CRD") + err := HubCluster.ImpersonateKubeClient.Create(ctx, &crd) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create CRD call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(string(statusErr.Status().Reason)).Should(Equal(fmt.Sprintf(crdStatusErrFormat, testUser, testGroups, crd.Name))) + }) + + It("should deny UPDATE operation on Fleet CRD for user not in system:masters group", func() { + var crd v1.CustomResourceDefinition + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: "memberclusters.fleet.azure.com"}, &crd)).Should(Succeed()) + + By("update labels in CRD") + labels := crd.GetLabels() + labels[testKey] = testValue + crd.SetLabels(labels) + + By("expecting denial of operation UPDATE of CRD") + err := HubCluster.ImpersonateKubeClient.Update(ctx, &crd) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update CRD call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(string(statusErr.Status().Reason)).Should(Equal(fmt.Sprintf(crdStatusErrFormat, testUser, testGroups, crd.Name))) + }) + + It("should deny DELETE operation on Fleet CRD for user not in system:masters group", func() { + crd := v1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "works.multicluster.x-k8s.io", + }, + } + By("expecting denial of operation Delete of CRD") + err := HubCluster.ImpersonateKubeClient.Delete(ctx, &crd) + var statusErr *k8sErrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Delete CRD call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) + Expect(string(statusErr.Status().Reason)).Should(Equal(fmt.Sprintf(crdStatusErrFormat, testUser, testGroups, crd.Name))) + }) + + It("should allow UPDATE operation on Fleet CRDs if user in system:masters group", func() { + var crd v1.CustomResourceDefinition + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: "memberclusters.fleet.azure.com"}, &crd)).Should(Succeed()) + + By("update labels in CRD") + labels := crd.GetLabels() + labels[testKey] = testValue + crd.SetLabels(labels) + + By("expecting denial of operation UPDATE of CRD") + // The user associated with KubeClient is kubernetes-admin in groups: [system:masters, system:authenticated] + Expect(HubCluster.KubeClient.Update(ctx, &crd)).To(Succeed()) + + By("remove new label added for test") + labels = crd.GetLabels() + delete(labels, testKey) + crd.SetLabels(labels) + }) + + It("should allow CREATE operation on Other CRDs", func() { + var crd v1.CustomResourceDefinition + Expect(utils.GetObjectFromManifest("./test/integration/manifests/resources/test_clonesets_crd.yaml", &crd)).Should(Succeed()) + + By("expecting error to be nil") + Expect(HubCluster.KubeClient.Create(ctx, &crd)).To(Succeed()) + + By("delete clone set CRD") + Expect(HubCluster.KubeClient.Delete(ctx, &crd)).To(Succeed()) + }) + }) +})