From e0ebc0d26233c626b1637bd5f17e42e3ed4d1803 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 18 Oct 2022 17:30:17 -0700 Subject: [PATCH 01/13] Unpick workload IT --- test/integration/cluster_placement_test.go | 60 +++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index 114cd41ac..327292f2a 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1023,8 +1023,66 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { markInternalMCJoined(clusterA) }) - XIt("Test cluster scoped resource change unpick by a placement", func() { + It("Test cluster scoped resource change unpick by a placement", func() { + crp = &fleetv1alpha1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: "resource-select", + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ + { + Group: rbacv1.GroupName, + Version: "v1", + Kind: ClusterRoleKind, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "fleet.azure.com/name": "test", + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, crp)).Should(Succeed()) + By("Select resource by label clusterResourcePlacement created") + + // verify that we have created work objects that contain the resource selected + verifyWorkObjects(crp, []string{ClusterRoleKind}, []*fleetv1alpha1.MemberCluster{&clusterA}) + + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)).Should(Succeed()) + By("Update CRP to not pick cluster role") + crp = &fleetv1alpha1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: "resource-select", + ResourceVersion: crp.ResourceVersion, + }, + Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ + { + Group: rbacv1.GroupName, + Version: "v1", + Kind: ClusterRoleKind, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "fleet.azure.com/env": "prod", + }, + }, + }, + }, + }, + } + Expect(k8sClient.Update(ctx, crp)).Should(Succeed()) + // verify that the work object created is not present anymore since we are not picking cluster role + nsName := fmt.Sprintf(utils.NamespaceNameFormat, clusterA.Name) + Eventually(func() bool { + var clusterWork workv1alpha1.Work + return apierrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{ + Name: crp.Name, + Namespace: nsName, + }, &clusterWork)) + }, timeout, interval).Should(BeTrue()) + By("Verified the work object is removed") }) It("Test a cluster scoped resource selected by multiple placements", func() { From 1b1ccdfca2673216020d4c8660d971d62f05b026 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Mon, 24 Oct 2022 12:21:52 -0700 Subject: [PATCH 02/13] Check CRP status --- test/integration/cluster_placement_test.go | 68 ++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index 327292f2a..e208c98a8 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1049,6 +1049,47 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { // verify that we have created work objects that contain the resource selected verifyWorkObjects(crp, []string{ClusterRoleKind}, []*fleetv1alpha1.MemberCluster{&clusterA}) + // Apply is Pending because work api controller is not being run for this test suite + fleetResourceIdentifier := fleetv1alpha1.ResourceIdentifier{ + Group: rbacv1.GroupName, + Version: "v1", + Kind: ClusterRoleKind, + Name: "test-cluster-role", + } + wantCRPStatus := fleetv1alpha1.ClusterResourcePlacementStatus{ + Conditions: []metav1.Condition{ + { + Type: string(fleetv1alpha1.ResourcePlacementConditionTypeScheduled), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + Reason: "ScheduleSucceeded", + }, + { + Type: string(fleetv1alpha1.ResourcePlacementStatusConditionTypeApplied), + Status: metav1.ConditionUnknown, + ObservedGeneration: 1, + Reason: "ApplyPending", + }, + }, + SelectedResources: []fleetv1alpha1.ResourceIdentifier{fleetResourceIdentifier}, + TargetClusters: []string{clusterA.Name}, + } + + crpStatusCmpOptions := []cmp.Option{ + cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "Message"), + cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }), + } + + Eventually(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp); err != nil { + return err + } + if diff := cmp.Diff(wantCRPStatus, crp.Status, crpStatusCmpOptions...); diff != "" { + return fmt.Errorf("CRP status(%s) mismatch (-want +got):\n%s", crp.Name, diff) + } + return nil + }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in %s cluster", clusterA.Name) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)).Should(Succeed()) By("Update CRP to not pick cluster role") crp = &fleetv1alpha1.ClusterResourcePlacement{ @@ -1083,6 +1124,33 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { }, &clusterWork)) }, timeout, interval).Should(BeTrue()) By("Verified the work object is removed") + + wantCRPStatus = fleetv1alpha1.ClusterResourcePlacementStatus{ + Conditions: []metav1.Condition{ + { + Type: string(fleetv1alpha1.ResourcePlacementConditionTypeScheduled), + Status: metav1.ConditionFalse, + ObservedGeneration: 2, + Reason: "ScheduleFailed", + }, + { + Type: string(fleetv1alpha1.ResourcePlacementStatusConditionTypeApplied), + Status: metav1.ConditionUnknown, + ObservedGeneration: 1, + Reason: "ApplyPending", + }, + }, + } + + Eventually(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp); err != nil { + return err + } + if diff := cmp.Diff(wantCRPStatus, crp.Status, crpStatusCmpOptions...); diff != "" { + return fmt.Errorf("CRP status(%s) mismatch (-want +got):\n%s", crp.Name, diff) + } + return nil + }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in %s cluster", clusterA.Name) }) It("Test a cluster scoped resource selected by multiple placements", func() { From 23830130f6864d342b3368dd6e8d9278830ae60d Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Mon, 24 Oct 2022 13:27:04 -0700 Subject: [PATCH 03/13] Change Cluster role instead of CRP --- test/integration/cluster_placement_test.go | 69 ++++++++++++++++------ 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index e208c98a8..aa270c394 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1091,28 +1091,33 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in %s cluster", clusterA.Name) Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)).Should(Succeed()) - By("Update CRP to not pick cluster role") - crp = &fleetv1alpha1.ClusterResourcePlacement{ + By("Update cluster role such that CRP doesn't pick it up") + cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: "resource-select", - ResourceVersion: crp.ResourceVersion, + Name: "test-cluster-role", + Labels: map[string]string{ + "fleet.azure.com/env": "prod", + }, }, - Spec: fleetv1alpha1.ClusterResourcePlacementSpec{ - ResourceSelectors: []fleetv1alpha1.ClusterResourceSelector{ - { - Group: rbacv1.GroupName, - Version: "v1", - Kind: ClusterRoleKind, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "fleet.azure.com/env": "prod", - }, - }, - }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"get", "list", "watch", "create", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, }, }, } - Expect(k8sClient.Update(ctx, crp)).Should(Succeed()) + Expect(k8sClient.Update(ctx, cr)).Should(Succeed()) // verify that the work object created is not present anymore since we are not picking cluster role nsName := fmt.Sprintf(utils.NamespaceNameFormat, clusterA.Name) @@ -1130,7 +1135,7 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { { Type: string(fleetv1alpha1.ResourcePlacementConditionTypeScheduled), Status: metav1.ConditionFalse, - ObservedGeneration: 2, + ObservedGeneration: 1, Reason: "ScheduleFailed", }, { @@ -1151,6 +1156,34 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { } return nil }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in %s cluster", clusterA.Name) + + By("revert update to cluster role since other tests are using it") + cr = &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-role", + Labels: map[string]string{ + "fleet.azure.com/name": "test", + }, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"get", "list", "watch", "create", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + }, + } + Expect(k8sClient.Update(ctx, cr)).Should(Succeed()) }) It("Test a cluster scoped resource selected by multiple placements", func() { From e611c620f7b29d672c6b962ed93a3bd6d50b9516 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Mon, 24 Oct 2022 13:33:16 -0700 Subject: [PATCH 04/13] Reduce LOC --- test/integration/cluster_placement_test.go | 27 +++------------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index aa270c394..af362616f 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1158,31 +1158,10 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in %s cluster", clusterA.Name) By("revert update to cluster role since other tests are using it") - cr = &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster-role", - Labels: map[string]string{ - "fleet.azure.com/name": "test", - }, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"get", "list", "watch", "create", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, + oldLabels := map[string]string{ + "fleet.azure.com/name": "test", } + cr.ObjectMeta.Labels = oldLabels Expect(k8sClient.Update(ctx, cr)).Should(Succeed()) }) From 94f98c18db3e429147b6c12ed571c83430849f47 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 26 Oct 2022 11:09:45 -0700 Subject: [PATCH 05/13] Use new ClusterRoleBinding --- test/integration/cluster_placement_test.go | 45 ++++++------------- .../resources/test_clusterrolebinding.yaml | 14 ++++++ test/integration/utils_test.go | 32 +++++++++---- 3 files changed, 50 insertions(+), 41 deletions(-) create mode 100644 test/integration/manifests/resources/test_clusterrolebinding.yaml diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index af362616f..6f35a1b6e 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1033,7 +1033,7 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { { Group: rbacv1.GroupName, Version: "v1", - Kind: ClusterRoleKind, + Kind: "ClusterRoleBinding", LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "fleet.azure.com/name": "test", @@ -1047,14 +1047,14 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { By("Select resource by label clusterResourcePlacement created") // verify that we have created work objects that contain the resource selected - verifyWorkObjects(crp, []string{ClusterRoleKind}, []*fleetv1alpha1.MemberCluster{&clusterA}) + verifyWorkObjects(crp, []string{"ClusterRoleBinding"}, []*fleetv1alpha1.MemberCluster{&clusterA}) // Apply is Pending because work api controller is not being run for this test suite fleetResourceIdentifier := fleetv1alpha1.ResourceIdentifier{ Group: rbacv1.GroupName, Version: "v1", - Kind: ClusterRoleKind, - Name: "test-cluster-role", + Kind: "ClusterRoleBinding", + Name: "test-cluster-role-binding", } wantCRPStatus := fleetv1alpha1.ClusterResourcePlacementStatus{ Conditions: []metav1.Condition{ @@ -1091,35 +1091,23 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in %s cluster", clusterA.Name) Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)).Should(Succeed()) - By("Update cluster role such that CRP doesn't pick it up") - cr := &rbacv1.ClusterRole{ + By("Update cluster role binding such that CRP doesn't pick it up") + crb := &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster-role", + Name: "test-cluster-role-binding", Labels: map[string]string{ "fleet.azure.com/env": "prod", }, }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"get", "list", "watch", "create", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: ClusterRoleKind, + Name: "test-cluster-role", }, } - Expect(k8sClient.Update(ctx, cr)).Should(Succeed()) + Expect(k8sClient.Update(ctx, crb)).Should(Succeed()) - // verify that the work object created is not present anymore since we are not picking cluster role + // verify that the work object created is not present anymore since we are not picking the namespace nsName := fmt.Sprintf(utils.NamespaceNameFormat, clusterA.Name) Eventually(func() bool { var clusterWork workv1alpha1.Work @@ -1156,13 +1144,6 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { } return nil }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in %s cluster", clusterA.Name) - - By("revert update to cluster role since other tests are using it") - oldLabels := map[string]string{ - "fleet.azure.com/name": "test", - } - cr.ObjectMeta.Labels = oldLabels - Expect(k8sClient.Update(ctx, cr)).Should(Succeed()) }) It("Test a cluster scoped resource selected by multiple placements", func() { diff --git a/test/integration/manifests/resources/test_clusterrolebinding.yaml b/test/integration/manifests/resources/test_clusterrolebinding.yaml new file mode 100644 index 000000000..2a15f0a62 --- /dev/null +++ b/test/integration/manifests/resources/test_clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: test-cluster-role-binding + labels: + fleet.azure.com/name: test +subjects: + - kind: User + name: jane # Name is case sensitive + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: test-cluster-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go index 3dd563582..1d4183aa5 100644 --- a/test/integration/utils_test.go +++ b/test/integration/utils_test.go @@ -49,14 +49,15 @@ var ( genericCodec runtime.Decoder // pre loaded test manifests - testClonesetCRD apiextensionsv1.CustomResourceDefinition - testClusterRole rbacv1.ClusterRole - testNameSpace corev1.Namespace - testCloneset kruisev1alpha1.CloneSet - testConfigMap corev1.ConfigMap - testSecret corev1.Secret - testService corev1.Service - testPdb policyv1.PodDisruptionBudget + testClonesetCRD apiextensionsv1.CustomResourceDefinition + testClusterRole rbacv1.ClusterRole + testClusterRoleBinding rbacv1.ClusterRoleBinding + testNameSpace corev1.Namespace + testCloneset kruisev1alpha1.CloneSet + testConfigMap corev1.ConfigMap + testSecret corev1.Secret + testService corev1.Service + testPdb policyv1.PodDisruptionBudget ) // GetObjectFromRawExtension returns an object decoded from the raw byte array @@ -80,7 +81,7 @@ func GetObjectFromManifest(relativeFilePath string, obj runtime.Object) { // applyTestManifests creates the test manifests in the hub cluster. // Here is the list, please do NOT change this list unless you know what you are doing. // ClusterScoped resource: -// Cloneset CRD, ClusterRole, Namespace +// Cloneset CRD, ClusterRole, ClusterRoleBinding Namespace // Namespaced resources: // Cloneset CR, Pdb, Configmap, Secret, Service. func applyTestManifests() { @@ -93,6 +94,10 @@ func applyTestManifests() { GetObjectFromManifest("manifests/resources/test_clusterrole.yaml", &testClusterRole) Expect(k8sClient.Create(ctx, &testClusterRole)).Should(Succeed()) + By("Create testClusterRoleBinding resource") + GetObjectFromManifest("manifests/resources/test_clusterrolebinding.yaml", &testClusterRoleBinding) + Expect(k8sClient.Create(ctx, &testClusterRoleBinding)).Should(Succeed()) + By("Create namespace") GetObjectFromManifest("manifests/resources/test_namespace.yaml", &testNameSpace) Expect(k8sClient.Create(ctx, &testNameSpace)).Should(Succeed()) @@ -123,6 +128,9 @@ func deleteTestManifests() { By("Delete testClusterRole resource") Expect(k8sClient.Delete(ctx, &testClusterRole)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + By("Delete testClusterRoleBinding resource") + Expect(k8sClient.Delete(ctx, &testClusterRoleBinding)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + By("Delete PodDisruptionBudget") Expect(k8sClient.Delete(ctx, &testPdb)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) @@ -168,6 +176,12 @@ func verifyManifest(manifest unstructured.Unstructured) { Expect(workClusterRole.GetName()).Should(Equal(testClusterRole.GetName())) Expect(workClusterRole.Rules).Should(Equal(testClusterRole.Rules)) + case "ClusterRoleBinding": + var workClusterRoleBinding rbacv1.ClusterRoleBinding + Expect(runtime.DefaultUnstructuredConverter.FromUnstructured(manifest.Object, &workClusterRoleBinding)).Should(Succeed()) + Expect(workClusterRoleBinding.GetName()).Should(Equal(testClusterRoleBinding.GetName())) + Expect(workClusterRoleBinding.RoleRef).Should(Equal(testClusterRoleBinding.RoleRef)) + case "Namespace": var workNameSpace corev1.Namespace Expect(runtime.DefaultUnstructuredConverter.FromUnstructured(manifest.Object, &workNameSpace)).Should(Succeed()) From 277b3be14abc6767c7df403b34dcfe1fc1d6061f Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 26 Oct 2022 11:13:18 -0700 Subject: [PATCH 06/13] Update comment --- test/integration/cluster_placement_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index 6f35a1b6e..5c9bb7dca 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1107,7 +1107,7 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { } Expect(k8sClient.Update(ctx, crb)).Should(Succeed()) - // verify that the work object created is not present anymore since we are not picking the namespace + // verify that the work object created is not present anymore since we are not picking the cluster role binding nsName := fmt.Sprintf(utils.NamespaceNameFormat, clusterA.Name) Eventually(func() bool { var clusterWork workv1alpha1.Work From 4c2ffd550f774158fd04f4adcb0f2698e7b79413 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 26 Oct 2022 11:29:18 -0700 Subject: [PATCH 07/13] fix comment --- .../manifests/resources/test_clusterrolebinding.yaml | 2 +- test/integration/utils_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/manifests/resources/test_clusterrolebinding.yaml b/test/integration/manifests/resources/test_clusterrolebinding.yaml index 2a15f0a62..0ab64d991 100644 --- a/test/integration/manifests/resources/test_clusterrolebinding.yaml +++ b/test/integration/manifests/resources/test_clusterrolebinding.yaml @@ -11,4 +11,4 @@ subjects: roleRef: kind: ClusterRole name: test-cluster-role - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go index 1d4183aa5..d2ddf3698 100644 --- a/test/integration/utils_test.go +++ b/test/integration/utils_test.go @@ -81,7 +81,7 @@ func GetObjectFromManifest(relativeFilePath string, obj runtime.Object) { // applyTestManifests creates the test manifests in the hub cluster. // Here is the list, please do NOT change this list unless you know what you are doing. // ClusterScoped resource: -// Cloneset CRD, ClusterRole, ClusterRoleBinding Namespace +// Cloneset CRD, ClusterRole, ClusterRoleBinding, Namespace. // Namespaced resources: // Cloneset CR, Pdb, Configmap, Secret, Service. func applyTestManifests() { From a93358da7ff06bf24880fcc5f1536fb1765a439a Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 26 Oct 2022 13:48:44 -0700 Subject: [PATCH 08/13] fix eventually messages --- test/integration/cluster_placement_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index 5c9bb7dca..4404dc0ae 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1088,7 +1088,7 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { return fmt.Errorf("CRP status(%s) mismatch (-want +got):\n%s", crp.Name, diff) } return nil - }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in %s cluster", clusterA.Name) + }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in hub cluster") Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)).Should(Succeed()) By("Update cluster role binding such that CRP doesn't pick it up") @@ -1143,7 +1143,7 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { return fmt.Errorf("CRP status(%s) mismatch (-want +got):\n%s", crp.Name, diff) } return nil - }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in %s cluster", clusterA.Name) + }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in hub cluster") }) It("Test a cluster scoped resource selected by multiple placements", func() { From b3183ef0896bdfadc82d93ce66ef6f4e456f0d92 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 26 Oct 2022 15:08:17 -0700 Subject: [PATCH 09/13] Inline creation of CRB --- test/integration/cluster_placement_test.go | 20 ++++++++++++++++++- .../resources/test_clusterrolebinding.yaml | 14 ------------- test/integration/utils_test.go | 10 ++-------- 3 files changed, 21 insertions(+), 23 deletions(-) delete mode 100644 test/integration/manifests/resources/test_clusterrolebinding.yaml diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index 4404dc0ae..b382d1185 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1024,6 +1024,23 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { }) It("Test cluster scoped resource change unpick by a placement", func() { + By("create cluster role binding") + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-role-binding", + Labels: map[string]string{ + "fleet.azure.com/name": "test", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: ClusterRoleKind, + Name: "test-cluster-role", + }, + } + Expect(k8sClient.Create(ctx, crb)).Should(Succeed()) + + By("create cluster resource placement") crp = &fleetv1alpha1.ClusterResourcePlacement{ ObjectMeta: metav1.ObjectMeta{ Name: "resource-select", @@ -1092,7 +1109,8 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)).Should(Succeed()) By("Update cluster role binding such that CRP doesn't pick it up") - crb := &rbacv1.ClusterRoleBinding{ + // changing label + crb = &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "test-cluster-role-binding", Labels: map[string]string{ diff --git a/test/integration/manifests/resources/test_clusterrolebinding.yaml b/test/integration/manifests/resources/test_clusterrolebinding.yaml deleted file mode 100644 index 0ab64d991..000000000 --- a/test/integration/manifests/resources/test_clusterrolebinding.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: test-cluster-role-binding - labels: - fleet.azure.com/name: test -subjects: - - kind: User - name: jane # Name is case sensitive - apiGroup: rbac.authorization.k8s.io -roleRef: - kind: ClusterRole - name: test-cluster-role - apiGroup: rbac.authorization.k8s.io diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go index d2ddf3698..9d2a53ce3 100644 --- a/test/integration/utils_test.go +++ b/test/integration/utils_test.go @@ -81,7 +81,7 @@ func GetObjectFromManifest(relativeFilePath string, obj runtime.Object) { // applyTestManifests creates the test manifests in the hub cluster. // Here is the list, please do NOT change this list unless you know what you are doing. // ClusterScoped resource: -// Cloneset CRD, ClusterRole, ClusterRoleBinding, Namespace. +// Cloneset CRD, ClusterRole, Namespace. // Namespaced resources: // Cloneset CR, Pdb, Configmap, Secret, Service. func applyTestManifests() { @@ -94,10 +94,6 @@ func applyTestManifests() { GetObjectFromManifest("manifests/resources/test_clusterrole.yaml", &testClusterRole) Expect(k8sClient.Create(ctx, &testClusterRole)).Should(Succeed()) - By("Create testClusterRoleBinding resource") - GetObjectFromManifest("manifests/resources/test_clusterrolebinding.yaml", &testClusterRoleBinding) - Expect(k8sClient.Create(ctx, &testClusterRoleBinding)).Should(Succeed()) - By("Create namespace") GetObjectFromManifest("manifests/resources/test_namespace.yaml", &testNameSpace) Expect(k8sClient.Create(ctx, &testNameSpace)).Should(Succeed()) @@ -128,9 +124,6 @@ func deleteTestManifests() { By("Delete testClusterRole resource") Expect(k8sClient.Delete(ctx, &testClusterRole)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) - By("Delete testClusterRoleBinding resource") - Expect(k8sClient.Delete(ctx, &testClusterRoleBinding)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) - By("Delete PodDisruptionBudget") Expect(k8sClient.Delete(ctx, &testPdb)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) @@ -178,6 +171,7 @@ func verifyManifest(manifest unstructured.Unstructured) { case "ClusterRoleBinding": var workClusterRoleBinding rbacv1.ClusterRoleBinding + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "test-cluster-role-binding"}, &testClusterRoleBinding)) Expect(runtime.DefaultUnstructuredConverter.FromUnstructured(manifest.Object, &workClusterRoleBinding)).Should(Succeed()) Expect(workClusterRoleBinding.GetName()).Should(Equal(testClusterRoleBinding.GetName())) Expect(workClusterRoleBinding.RoleRef).Should(Equal(testClusterRoleBinding.RoleRef)) From b9c5460789d9b81219d87903835496e23fe29e6c Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 26 Oct 2022 15:10:50 -0700 Subject: [PATCH 10/13] revert change --- test/integration/utils_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go index 9d2a53ce3..458d4d80d 100644 --- a/test/integration/utils_test.go +++ b/test/integration/utils_test.go @@ -81,7 +81,7 @@ func GetObjectFromManifest(relativeFilePath string, obj runtime.Object) { // applyTestManifests creates the test manifests in the hub cluster. // Here is the list, please do NOT change this list unless you know what you are doing. // ClusterScoped resource: -// Cloneset CRD, ClusterRole, Namespace. +// Cloneset CRD, ClusterRole, Namespace // Namespaced resources: // Cloneset CR, Pdb, Configmap, Secret, Service. func applyTestManifests() { From 091ea85051f89a2d5954145c370269f1fca71266 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 27 Oct 2022 11:33:58 -0700 Subject: [PATCH 11/13] Delete CRB inline --- test/integration/cluster_placement_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index b382d1185..751e451e7 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1162,6 +1162,9 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { } return nil }, timeout, interval).Should(Succeed(), "Failed to compare actual and expected CRP status in hub cluster") + + By("Delete cluster role binding") + Expect(k8sClient.Delete(ctx, crb)).Should(Succeed()) }) It("Test a cluster scoped resource selected by multiple placements", func() { From cc05f226e85979dd4a801fab94d43e9818a5b1fa Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 27 Oct 2022 14:49:10 -0700 Subject: [PATCH 12/13] address comments --- test/integration/cluster_placement_test.go | 25 ++++++++++------------ test/integration/utils_test.go | 24 +++++++-------------- 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index 751e451e7..1a558f2b0 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1063,8 +1063,15 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { Expect(k8sClient.Create(ctx, crp)).Should(Succeed()) By("Select resource by label clusterResourcePlacement created") - // verify that we have created work objects that contain the resource selected - verifyWorkObjects(crp, []string{"ClusterRoleBinding"}, []*fleetv1alpha1.MemberCluster{&clusterA}) + // verify that we have created the work object + var clusterWork workv1alpha1.Work + Eventually(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{ + Name: crp.Name, Namespace: fmt.Sprintf(utils.NamespaceNameFormat, clusterA.Name)}, &clusterWork); err != nil { + return err + } + return nil + }, timeout, interval).Should(Succeed(), "Failed to retrieve %s work", crp.Name) // Apply is Pending because work api controller is not being run for this test suite fleetResourceIdentifier := fleetv1alpha1.ResourceIdentifier{ @@ -1110,18 +1117,8 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)).Should(Succeed()) By("Update cluster role binding such that CRP doesn't pick it up") // changing label - crb = &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster-role-binding", - Labels: map[string]string{ - "fleet.azure.com/env": "prod", - }, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: rbacv1.GroupName, - Kind: ClusterRoleKind, - Name: "test-cluster-role", - }, + crb.ObjectMeta.Labels = map[string]string{ + "fleet.azure.com/env": "prod", } Expect(k8sClient.Update(ctx, crb)).Should(Succeed()) diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go index 458d4d80d..3dd563582 100644 --- a/test/integration/utils_test.go +++ b/test/integration/utils_test.go @@ -49,15 +49,14 @@ var ( genericCodec runtime.Decoder // pre loaded test manifests - testClonesetCRD apiextensionsv1.CustomResourceDefinition - testClusterRole rbacv1.ClusterRole - testClusterRoleBinding rbacv1.ClusterRoleBinding - testNameSpace corev1.Namespace - testCloneset kruisev1alpha1.CloneSet - testConfigMap corev1.ConfigMap - testSecret corev1.Secret - testService corev1.Service - testPdb policyv1.PodDisruptionBudget + testClonesetCRD apiextensionsv1.CustomResourceDefinition + testClusterRole rbacv1.ClusterRole + testNameSpace corev1.Namespace + testCloneset kruisev1alpha1.CloneSet + testConfigMap corev1.ConfigMap + testSecret corev1.Secret + testService corev1.Service + testPdb policyv1.PodDisruptionBudget ) // GetObjectFromRawExtension returns an object decoded from the raw byte array @@ -169,13 +168,6 @@ func verifyManifest(manifest unstructured.Unstructured) { Expect(workClusterRole.GetName()).Should(Equal(testClusterRole.GetName())) Expect(workClusterRole.Rules).Should(Equal(testClusterRole.Rules)) - case "ClusterRoleBinding": - var workClusterRoleBinding rbacv1.ClusterRoleBinding - Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "test-cluster-role-binding"}, &testClusterRoleBinding)) - Expect(runtime.DefaultUnstructuredConverter.FromUnstructured(manifest.Object, &workClusterRoleBinding)).Should(Succeed()) - Expect(workClusterRoleBinding.GetName()).Should(Equal(testClusterRoleBinding.GetName())) - Expect(workClusterRoleBinding.RoleRef).Should(Equal(testClusterRoleBinding.RoleRef)) - case "Namespace": var workNameSpace corev1.Namespace Expect(runtime.DefaultUnstructuredConverter.FromUnstructured(manifest.Object, &workNameSpace)).Should(Succeed()) From f527d3ea1cfe38f01234aa95cc8bcbda9b950e53 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 27 Oct 2022 16:36:55 -0700 Subject: [PATCH 13/13] Remove comment --- test/integration/cluster_placement_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/cluster_placement_test.go b/test/integration/cluster_placement_test.go index 1a558f2b0..efe61f052 100644 --- a/test/integration/cluster_placement_test.go +++ b/test/integration/cluster_placement_test.go @@ -1116,7 +1116,6 @@ var _ = Describe("Test Cluster Resource Placement Controller", func() { Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)).Should(Succeed()) By("Update cluster role binding such that CRP doesn't pick it up") - // changing label crb.ObjectMeta.Labels = map[string]string{ "fleet.azure.com/env": "prod", }