diff --git a/test/e2e/join_leave_member_test.go b/test/e2e/join_leave_member_test.go index eab4b7b62..63aff8532 100644 --- a/test/e2e/join_leave_member_test.go +++ b/test/e2e/join_leave_member_test.go @@ -9,11 +9,8 @@ import ( "fmt" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/pkg/utils" @@ -25,57 +22,51 @@ var _ = Describe("Join/leave member cluster testing", func() { var sa *corev1.ServiceAccount var memberNS *corev1.Namespace var imc *v1alpha1.InternalMemberCluster + var ctx context.Context BeforeEach(func() { - memberNS = testutils.NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName)) + ctx = context.TODO() + memberNS = testutils.NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName), nil) By("prepare resources in member cluster") // create testing NS in member cluster - testutils.CreateNamespace(*MemberCluster, memberNS) + testutils.CreateNamespace(ctx, *MemberCluster, memberNS) sa = testutils.NewServiceAccount(MemberCluster.ClusterName, memberNS.Name) - testutils.CreateServiceAccount(*MemberCluster, sa) + testutils.CreateServiceAccount(ctx, *MemberCluster, sa) By("deploy member cluster in the hub cluster") mc = testutils.NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin) - testutils.CreateMemberCluster(*HubCluster, mc) + testutils.CreateMemberCluster(ctx, *HubCluster, mc) By("check if internal member cluster created in the hub cluster") imc = testutils.NewInternalMemberCluster(MemberCluster.ClusterName, memberNS.Name) - testutils.WaitInternalMemberCluster(*HubCluster, imc) + testutils.WaitInternalMemberCluster(ctx, *HubCluster, imc) By("check if member cluster is marked as readyToJoin") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionTrue, 3*testutils.PollTimeout) + testutils.WaitConditionMemberCluster(ctx, *HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionTrue, 3*testutils.PollTimeout) }) AfterEach(func() { - testutils.DeleteNamespace(*MemberCluster, memberNS) - Eventually(func() bool { - err := MemberCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: memberNS.Name, Namespace: ""}, memberNS) - return apierrors.IsNotFound(err) - }, testutils.PollTimeout, testutils.PollInterval).Should(Equal(true)) - testutils.DeleteMemberCluster(*HubCluster, mc) - Eventually(func() bool { - err := HubCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: memberNS.Name, Namespace: ""}, memberNS) - return apierrors.IsNotFound(err) - }, testutils.PollTimeout, testutils.PollInterval).Should(Equal(true)) + testutils.DeleteNamespace(ctx, *MemberCluster, memberNS) + testutils.DeleteMemberCluster(ctx, *HubCluster, mc) }) It("Join & Leave flow is successful ", func() { By("check if internal member cluster condition is updated to Joined") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, 3*testutils.PollTimeout) + testutils.WaitConditionInternalMemberCluster(ctx, *HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, 3*testutils.PollTimeout) By("check if member cluster condition is updated to Joined") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionTrue, 3*testutils.PollTimeout) + testutils.WaitConditionMemberCluster(ctx, *HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionTrue, 3*testutils.PollTimeout) By("update member cluster in the hub cluster") - testutils.UpdateMemberClusterState(*HubCluster, mc, v1alpha1.ClusterStateLeave) + testutils.UpdateMemberClusterState(ctx, *HubCluster, mc, v1alpha1.ClusterStateLeave) By("check if internal member cluster condition is updated to Left") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionFalse, 3*testutils.PollTimeout) + testutils.WaitConditionInternalMemberCluster(ctx, *HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionFalse, 3*testutils.PollTimeout) By("check if member cluster is marked as notReadyToJoin") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionFalse, 3*testutils.PollTimeout) + testutils.WaitConditionMemberCluster(ctx, *HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionFalse, 3*testutils.PollTimeout) By("check if member cluster condition is updated to Left") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionFalse, 3*testutils.PollTimeout) + testutils.WaitConditionMemberCluster(ctx, *HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionFalse, 3*testutils.PollTimeout) }) }) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index 2b58a9a96..e063d58c8 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -9,7 +9,8 @@ import ( "fmt" "time" - "github.com/onsi/ginkgo/v2" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -70,132 +71,250 @@ func NewServiceAccount(name, namespace string) *corev1.ServiceAccount { } // NewNamespace returns a new namespace. -func NewNamespace(name string) *corev1.Namespace { +func NewNamespace(name string, labels map[string]string) *corev1.Namespace { return &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: name, + Labels: labels, }, } } // CreateMemberCluster creates MemberCluster and waits for MemberCluster to exist in the hub cluster. -func CreateMemberCluster(cluster framework.Cluster, mc *v1alpha1.MemberCluster) { - ginkgo.By(fmt.Sprintf("Creating MemberCluster(%s)", mc.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), mc) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - }) - klog.Infof("Waiting for MemberCluster(%s) to be synced", mc.Name) +func CreateMemberCluster(ctx context.Context, cluster framework.Cluster, mc *v1alpha1.MemberCluster) { + klog.Infof("Creating MemberCluster(%s) in %s cluster", mc.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Create(ctx, mc)).Should(gomega.Succeed(), "Failed to create member cluster %s in %s cluster", mc.Name, cluster.ClusterName) + klog.Infof("Waiting for MemberCluster(%s) to be synced in %s cluster", mc.Name, cluster.ClusterName) gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: mc.Name, Namespace: ""}, mc) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) + return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for member cluster to be created in %s cluster", mc.Name, cluster.ClusterName) } // UpdateMemberClusterState updates MemberCluster in the hub cluster. -func UpdateMemberClusterState(cluster framework.Cluster, mc *v1alpha1.MemberCluster, state v1alpha1.ClusterState) { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: mc.Name, Namespace: ""}, mc) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) +func UpdateMemberClusterState(ctx context.Context, cluster framework.Cluster, mc *v1alpha1.MemberCluster, state v1alpha1.ClusterState) { + err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc) + gomega.Expect(err).Should(gomega.Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, cluster.ClusterName) mc.Spec.State = state - err = cluster.KubeClient.Update(context.TODO(), mc) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + err = cluster.KubeClient.Update(ctx, mc) + gomega.Expect(err).Should(gomega.Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, cluster.ClusterName) } // DeleteMemberCluster deletes MemberCluster in the hub cluster. -func DeleteMemberCluster(cluster framework.Cluster, mc *v1alpha1.MemberCluster) { - ginkgo.By(fmt.Sprintf("Deleting MemberCluster(%s)", mc.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), mc) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - }) +func DeleteMemberCluster(ctx context.Context, cluster framework.Cluster, mc *v1alpha1.MemberCluster) { + klog.Infof("Deleting MemberCluster(%s) in %s cluster", mc.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Delete(ctx, mc)).Should(gomega.Succeed(), "Failed to delete member cluster %s in %s cluster", mc.Name, cluster.ClusterName) + klog.Infof("Waiting for MemberCluster (%s) to be deleted in %s cluster", mc.Name, cluster.ClusterName) + gomega.Eventually(func() bool { + return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)) + }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for member cluster %s to be deleted in %s cluster", mc.Name, cluster.ClusterName) } // WaitConditionMemberCluster waits for MemberCluster to present on th hub cluster with a specific condition. -func WaitConditionMemberCluster(cluster framework.Cluster, mc *v1alpha1.MemberCluster, conditionType v1alpha1.MemberClusterConditionType, status metav1.ConditionStatus, customTimeout time.Duration) { +func WaitConditionMemberCluster(ctx context.Context, cluster framework.Cluster, mc *v1alpha1.MemberCluster, conditionType v1alpha1.MemberClusterConditionType, status metav1.ConditionStatus, customTimeout time.Duration) { klog.Infof("Waiting for MemberCluster(%s) condition(%s) status(%s) to be synced", mc.Name, conditionType, status) gomega.Eventually(func() bool { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: mc.Name, Namespace: ""}, mc) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name, Namespace: ""}, mc) + if err != nil { + return false + } cond := mc.GetCondition(string(conditionType)) return cond != nil && cond.Status == status - }, customTimeout, PollInterval).Should(gomega.Equal(true)) + }, customTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for member cluster %s to have condition %s with status %s", mc.Name, string(conditionType), status) } -// WaitInternalMemberCluster waits for InternalMemberCluster to present on th hub cluster. -func WaitInternalMemberCluster(cluster framework.Cluster, imc *v1alpha1.InternalMemberCluster) { +// WaitInternalMemberCluster waits for InternalMemberCluster to present on the hub cluster. +func WaitInternalMemberCluster(ctx context.Context, cluster framework.Cluster, imc *v1alpha1.InternalMemberCluster) { klog.Infof("Waiting for InternalMemberCluster(%s) to be synced in the %s cluster", imc.Name, cluster.ClusterName) gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) + return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for internal member cluster %s to be synced in %s cluster", imc.Name, cluster.ClusterName) } // WaitConditionInternalMemberCluster waits for InternalMemberCluster to present on the hub cluster with a specific condition. // Allowing custom timeout as for join cond it needs longer than defined PollTimeout for the member agent to finish joining. -func WaitConditionInternalMemberCluster(cluster framework.Cluster, imc *v1alpha1.InternalMemberCluster, conditionType v1alpha1.AgentConditionType, status metav1.ConditionStatus, customTimeout time.Duration) { +func WaitConditionInternalMemberCluster(ctx context.Context, cluster framework.Cluster, imc *v1alpha1.InternalMemberCluster, conditionType v1alpha1.AgentConditionType, status metav1.ConditionStatus, customTimeout time.Duration) { klog.Infof("Waiting for InternalMemberCluster(%s) condition(%s) status(%s) to be synced in the %s cluster", imc.Name, conditionType, status, cluster.ClusterName) gomega.Eventually(func() bool { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc) + if err != nil { + return false + } cond := imc.GetConditionWithType(v1alpha1.MemberAgent, string(conditionType)) return cond != nil && cond.Status == status - }, customTimeout, PollInterval).Should(gomega.Equal(true)) + }, customTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for internal member cluster %s to have condition %s with status %s", imc.Name, string(conditionType), status) } // CreateClusterRole create cluster role in the hub cluster. -func CreateClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { - ginkgo.By(fmt.Sprintf("Creating ClusterRole (%s)", cr.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), cr) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - }) +func CreateClusterRole(ctx context.Context, cluster framework.Cluster, cr *rbacv1.ClusterRole) { + klog.Infof("Creating ClusterRole (%s) in %s cluster", cr.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Create(ctx, cr)).Should(gomega.Succeed(), "Failed to create cluster role %s in %s cluster", cr.Name, cluster.ClusterName) } -// WaitClusterRole waits for cluster roles to be created. -func WaitClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { - klog.Infof("Waiting for ClusterRole(%s) to be synced", cr.Name) +// GetClusterRole retrieves cluster role. +func GetClusterRole(ctx context.Context, cluster framework.Cluster, cr *rbacv1.ClusterRole) { + klog.Infof("Get Role(%s) in %s cluster", cr.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)) +} + +// UpdateClusterRole updates cluster role in hub cluster. +func UpdateClusterRole(ctx context.Context, cluster framework.Cluster, cr *rbacv1.ClusterRole) { + klog.Infof("Updating ClusterRole (%s) in %s cluster", cr.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Update(ctx, cr)).Should(gomega.Succeed(), "Failed to update cluster role %s in %s cluster", cr.Name, cluster.ClusterName) +} + +// WaitUpdateClusterRoleLabels waits for cluster role to be updated to new labels. +func WaitUpdateClusterRoleLabels(ctx context.Context, cluster framework.Cluster, clusterRole, newClusterRole *rbacv1.ClusterRole) { + klog.Infof("Waiting for ClusterRole(%s) to be updated in %s cluster", clusterRole.Name, cluster.ClusterName) gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) + err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: clusterRole.Name}, clusterRole) + if err != nil { + return err + } + diff := cmp.Diff(clusterRole.Labels, newClusterRole.Labels) + if diff != "" { + return fmt.Errorf("Role(%s) rules mismatch (-want +got):\n%s", clusterRole.Name, diff) + } + return nil + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for cluster role %s to be updated in %s cluster", clusterRole.Name, cluster.ClusterName) } // DeleteClusterRole deletes cluster role on cluster. -func DeleteClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { - ginkgo.By(fmt.Sprintf("Deleting ClusterRole(%s)", cr.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), cr) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - }) +func DeleteClusterRole(ctx context.Context, cluster framework.Cluster, cr *rbacv1.ClusterRole) { + klog.Infof("Deleting ClusterRole(%s) in %s cluster", cr.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Delete(ctx, cr)).Should(gomega.Succeed(), "Failed to delete cluster role %s in %s cluster", cr.Name, cluster.ClusterName) + klog.Infof("Waiting for Cluster Role(%s) to be deleted", cr.Name) + gomega.Eventually(func() bool { + return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)) + }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", cr.Name, cluster.ClusterName) } -// CreateClusterResourcePlacement created ClusterResourcePlacement and waits for ClusterResourcePlacement to exist in hub cluster. -func CreateClusterResourcePlacement(cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { - ginkgo.By(fmt.Sprintf("Creating ClusterResourcePlacement(%s)", crp.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), crp) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - }) - klog.Infof("Waiting for ClusterResourcePlacement(%s) to be synced", crp.Name) +// CreateRole creates role in hub cluster. +func CreateRole(ctx context.Context, cluster framework.Cluster, r *rbacv1.Role) { + klog.Infof("Creating Role(%s) in %s cluster", r.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Create(ctx, r)).Should(gomega.Succeed(), "Failed to create role %s in %s cluster", r.Name, cluster.ClusterName) +} + +// UpdateRole updates role in hub cluster. +func UpdateRole(ctx context.Context, cluster framework.Cluster, r *rbacv1.Role) { + klog.Infof("Updating Role(%s) in %s cluster", r.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Update(ctx, r)).Should(gomega.Succeed(), "Failed to update role %s in %s cluster", r.Name, cluster.ClusterName) +} + +// WaitCreateRole waits for role to be created. +func WaitCreateRole(ctx context.Context, cluster framework.Cluster, r *rbacv1.Role) { + klog.Infof("Waiting for Role(%s) to be created in %s cluster", r.Name, cluster.ClusterName) gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: crp.Name, Namespace: ""}, crp) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) + return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: r.Name, Namespace: r.Namespace}, r) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for role %s in %s cluster", r.Name, cluster.ClusterName) +} + +// GetRole retrieves role. +func GetRole(ctx context.Context, cluster framework.Cluster, r *rbacv1.Role) { + klog.Infof("Get Role(%s) in %s cluster", r.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: r.Name, Namespace: r.Namespace}, r)) } -// WaitConditionClusterResourcePlacement waits for ClusterResourcePlacement to present on th hub cluster with a specific condition. -func WaitConditionClusterResourcePlacement(cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement, - conditionName string, status metav1.ConditionStatus, customTimeout time.Duration) { - klog.Infof("Waiting for ClusterResourcePlacement(%s) condition(%s) status(%s) to be synced", crp.Name, conditionName, status) +// WaitDeleteRole waits for role to be deleted. +func WaitDeleteRole(ctx context.Context, cluster framework.Cluster, r *rbacv1.Role) { + klog.Infof("Waiting for Role(%s) to be updated in %s cluster", r.Name, cluster.ClusterName) gomega.Eventually(func() bool { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: crp.Name, Namespace: ""}, crp) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - cond := crp.GetCondition(conditionName) - return cond != nil && cond.Status == status - }, customTimeout, PollInterval).Should(gomega.Equal(true)) + return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: r.Name, Namespace: r.Namespace}, r)) + }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for role %s to be deleted in %s cluster", r.Name, cluster.ClusterName) +} + +// WaitUpdateRoleRules waits for role to be updated to new rules. +func WaitUpdateRoleRules(ctx context.Context, cluster framework.Cluster, role, newRole *rbacv1.Role) { + klog.Infof("Waiting for Role(%s) to be updated in %s cluster", role.Name, cluster.ClusterName) + gomega.Eventually(func() error { + err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, role) + if err != nil { + return err + } + diff := cmp.Diff(role.Rules, newRole.Rules) + if diff != "" { + return fmt.Errorf("Role(%s) rules mismatch (-want +got):\n%s", role.Name, diff) + } + return nil + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for role %s to be updated in %s cluster", role.Name, cluster.ClusterName) +} + +// DeleteRole deletes cluster role on cluster. +func DeleteRole(ctx context.Context, cluster framework.Cluster, r *rbacv1.Role) { + klog.Infof("Deleting Role(%s) in %s cluster", r.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Delete(ctx, r)).Should(gomega.Succeed(), "Failed to delete role %s in %s cluster", r.Name, cluster.ClusterName) + klog.Infof("Waiting for Role(%s) to be deleted", r.Name) + gomega.Eventually(func() bool { + return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: r.Name, Namespace: r.Namespace}, r)) + }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for role %s to be deleted in %s cluster", r.Name, cluster.ClusterName) +} + +// CreateRoleBinding creates role binding in hub cluster. +func CreateRoleBinding(ctx context.Context, cluster framework.Cluster, rb *rbacv1.RoleBinding) { + klog.Infof("Creating Role Binding (%s) in %s cluster", rb.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Create(ctx, rb)).Should(gomega.Succeed(), "Failed to create role binding %s in %s cluster", rb.Name, cluster.ClusterName) +} + +// GetRoleBinding retrieves role binding. +func GetRoleBinding(ctx context.Context, cluster framework.Cluster, rb *rbacv1.RoleBinding) { + klog.Infof("Get RoleBinding(%s) in %s cluster", rb.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: rb.Name, Namespace: rb.Namespace}, rb)) +} + +// CreateClusterResourcePlacement created ClusterResourcePlacement and waits for ClusterResourcePlacement to exist in hub cluster. +func CreateClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { + klog.Infof("Creating ClusterResourcePlacement(%s) in %s cluster", crp.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Create(ctx, crp)).Should(gomega.Succeed()) + klog.Infof("Waiting for ClusterResourcePlacement(%s) to be created", crp.Name) + gomega.Eventually(func() error { + return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, cluster.ClusterName) +} + +// WaitCreateClusterResourcePlacement waits for ClusterResourcePlacement to present on th hub cluster with a specific status. +func WaitCreateClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement, conditionType []string, + conditionStatus []metav1.ConditionStatus, selectedResources []v1alpha1.ResourceIdentifier, targetClusters []string, customTimeout time.Duration) { + klog.Infof("Waiting for ClusterResourcePlacement(%s) status to be synced in %s cluster", crp.Name, cluster.ClusterName) + gomega.Eventually(func() error { + err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp) + if err != nil { + return err + } + for i := range conditionType { + condition := crp.GetCondition(conditionType[i]) + if condition != nil && condition.Status != conditionStatus[i] { + return fmt.Errorf("cluster resource placement(%s) failed to %s resources", crp.Name, conditionType) + } + } + less := func(a, b v1alpha1.ResourceIdentifier) bool { return a.Name < b.Name } + selectedResourceDiff := cmp.Diff(selectedResources, crp.Status.SelectedResources, cmpopts.SortSlices(less)) + if selectedResourceDiff != "" { + return fmt.Errorf("cluster resource placment(%s) selected resources mismatch (-want +got):\n%s", crp.Name, selectedResourceDiff) + } + targetClustersDiff := cmp.Diff(targetClusters, crp.Status.TargetClusters) + if targetClustersDiff != "" { + return fmt.Errorf("cluster resource placment(%s) target clusters mismatch (-want +got):\n%s", crp.Name, targetClustersDiff) + } + if crp.Status.FailedResourcePlacements != nil { + return fmt.Errorf("cluster resource placement(%s) failed resource placements should be nil", crp.Name) + } + return nil + }, customTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for cluster resource placement %s to be created in %s cluster", crp.Name, cluster.ClusterName) +} + +// UpdateClusterResourcePlacement updates cluster resource placement in hub cluster. +func UpdateClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { + klog.Infof("Updating Cluster Resource Placement(%s) in %s cluster", crp.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Update(ctx, crp)).Should(gomega.Succeed(), "Failed to update cluster resource placement %s in %s cluster", crp.Name, cluster.ClusterName) } // DeleteClusterResourcePlacement is used delete ClusterResourcePlacement on the hub cluster. -func DeleteClusterResourcePlacement(cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { - ginkgo.By(fmt.Sprintf("Deleting ClusterResourcePlacement(%s)", crp.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), crp) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - }) +func DeleteClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { + klog.Infof("Deleting ClusterResourcePlacement(%s) in %s cluster", crp.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Delete(ctx, crp)).Should(gomega.Succeed(), "Failed to delete cluster resource placement %s in %s cluster", crp.Name, cluster.ClusterName) + klog.Infof("Waiting for Cluster Resource Placement(%s) to be deleted", crp.Name) + gomega.Eventually(func() bool { + return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)) + }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for cluster resource placement %s to be deleted in %s cluster", crp.Name, cluster.ClusterName) } // WaitWork waits for Work to be present on the hub cluster. @@ -209,40 +328,33 @@ func WaitWork(cluster framework.Cluster, workName, workNamespace string) { } // CreateNamespace create namespace and waits for namespace to exist. -func CreateNamespace(cluster framework.Cluster, ns *corev1.Namespace) { - ginkgo.By(fmt.Sprintf("Creating Namespace(%s)", ns.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), ns) - gomega.Expect(err).Should(gomega.Succeed()) - }) - klog.Infof("Waiting for Namespace(%s) to be synced", ns.Name) +func CreateNamespace(ctx context.Context, cluster framework.Cluster, ns *corev1.Namespace) { + klog.Infof("Creating Namespace(%s) in %s cluster", ns.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Create(ctx, ns)).Should(gomega.Succeed(), "Failed to create namespace %s in %s cluster", ns.Name, cluster.ClusterName) + klog.Infof("Waiting for Namespace(%s) to be created in %s cluster", ns.Name) gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: ns.Name, Namespace: ""}, ns) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) + return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: ns.Name}, ns) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for namespace %s to be created in %s cluster", ns.Name, cluster.ClusterName) } -// DeleteNamespace delete namespace. -func DeleteNamespace(cluster framework.Cluster, ns *corev1.Namespace) { - ginkgo.By(fmt.Sprintf("Deleting Namespace(%s)", ns.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), ns) - if err != nil && !apierrors.IsNotFound(err) { - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - } - }) +// GetNamespace retrieves namespace. +func GetNamespace(ctx context.Context, cluster framework.Cluster, ns *corev1.Namespace) { + klog.Infof("Get Namespace(%s) in %s cluster", ns.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: ns.Name}, ns)) } -// CreateServiceAccount create serviceaccount. -func CreateServiceAccount(cluster framework.Cluster, sa *corev1.ServiceAccount) { - ginkgo.By(fmt.Sprintf("Creating ServiceAccount(%s)", sa.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), sa) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - }) +// DeleteNamespace delete namespace. +func DeleteNamespace(ctx context.Context, cluster framework.Cluster, ns *corev1.Namespace) { + klog.Infof("Deleting Namespace(%s) in %s cluster", ns.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Delete(context.TODO(), ns)).Should(gomega.Succeed(), "Failed to delete namespace %s in %s cluster", ns.Name, cluster.ClusterName) + klog.Infof("Waiting for Namespace(%s) to be deleted in %s cluster", ns.Name, cluster.ClusterName) + gomega.Eventually(func() bool { + return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: ns.Name}, ns)) + }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for namespace %s to be deleted in %s cluster", ns.Name, cluster.ClusterName) } -// DeleteServiceAccount delete serviceaccount. -func DeleteServiceAccount(cluster framework.Cluster, sa *corev1.ServiceAccount) { - ginkgo.By(fmt.Sprintf("Delete ServiceAccount(%s)", sa.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), sa) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - }) +// CreateServiceAccount creates service account. +func CreateServiceAccount(ctx context.Context, cluster framework.Cluster, sa *corev1.ServiceAccount) { + klog.Infof("Creating ServiceAccount(%s) in %s cluster", sa.Name, cluster.ClusterName) + gomega.Expect(cluster.KubeClient.Create(ctx, sa)).Should(gomega.Succeed(), "Failed to create service account %s in %s cluster") } diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index a5d482d4d..ae1ce0246 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -10,12 +10,9 @@ import ( "fmt" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/pkg/utils" @@ -23,54 +20,448 @@ import ( ) var _ = Describe("workload orchestration testing", func() { - var mc *v1alpha1.MemberCluster - var sa *corev1.ServiceAccount - var memberNS *corev1.Namespace - var imc *v1alpha1.InternalMemberCluster - var cr *rbacv1.ClusterRole - var crp *v1alpha1.ClusterResourcePlacement + var ( + mc *v1alpha1.MemberCluster + sa *corev1.ServiceAccount + memberNS *corev1.Namespace + imc *v1alpha1.InternalMemberCluster + ctx context.Context + ) + + labelKey := "fleet.azure.com/name" + labelValue := "test" + + selectedResource1 := v1alpha1.ResourceIdentifier{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: "test-cluster-role", + } + selectedResource2 := v1alpha1.ResourceIdentifier{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "Role", + Name: "test-pod-reader", + Namespace: "test-namespace", + } + selectedResource3 := v1alpha1.ResourceIdentifier{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "RoleBinding", + Name: "read-pods", + Namespace: "test-namespace", + } + selectedResource4 := v1alpha1.ResourceIdentifier{ + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + } + selectedResource5 := v1alpha1.ResourceIdentifier{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "Role", + Name: "test-service-reader", + Namespace: "test-namespace", + } + selectedResource6 := v1alpha1.ResourceIdentifier{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: "new-cluster-role", + } + + conditionStatus := []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionTrue} + conditionType := []string{string(v1alpha1.ResourcePlacementConditionTypeScheduled), string(v1alpha1.ResourcePlacementStatusConditionTypeApplied)} + targetClusters := []string{"kind-member-testing"} BeforeEach(func() { - memberNS = testutils.NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName)) + ctx = context.TODO() + memberNS = testutils.NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName), nil) By("prepare resources in member cluster") // create testing NS in member cluster - testutils.CreateNamespace(*MemberCluster, memberNS) + testutils.CreateNamespace(ctx, *MemberCluster, memberNS) sa = testutils.NewServiceAccount(MemberCluster.ClusterName, memberNS.Name) - testutils.CreateServiceAccount(*MemberCluster, sa) + testutils.CreateServiceAccount(ctx, *MemberCluster, sa) By("deploy member cluster in the hub cluster") mc = testutils.NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin) - testutils.CreateMemberCluster(*HubCluster, mc) + testutils.CreateMemberCluster(ctx, *HubCluster, mc) By("check if internal member cluster created in the hub cluster") imc = testutils.NewInternalMemberCluster(MemberCluster.ClusterName, memberNS.Name) - testutils.WaitInternalMemberCluster(*HubCluster, imc) + testutils.WaitInternalMemberCluster(ctx, *HubCluster, imc) By("check if internal member cluster condition is updated to Joined") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, 3*testutils.PollTimeout) + testutils.WaitConditionInternalMemberCluster(ctx, *HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, 3*testutils.PollTimeout) By("check if member cluster condition is updated to Joined") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionTrue, 3*testutils.PollTimeout) + testutils.WaitConditionMemberCluster(ctx, *HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionTrue, 3*testutils.PollTimeout) }) AfterEach(func() { - testutils.DeleteMemberCluster(*HubCluster, mc) - Eventually(func() bool { - err := HubCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: memberNS.Name, Namespace: ""}, memberNS) - return apierrors.IsNotFound(err) - }, testutils.PollTimeout, testutils.PollInterval).Should(Equal(true)) - testutils.DeleteNamespace(*MemberCluster, memberNS) - Eventually(func() bool { - err := MemberCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: memberNS.Name, Namespace: ""}, memberNS) - return apierrors.IsNotFound(err) - }, testutils.PollTimeout, testutils.PollInterval).Should(Equal(true)) + testutils.DeleteMemberCluster(ctx, *HubCluster, mc) + testutils.DeleteNamespace(ctx, *MemberCluster, memberNS) + }) + + It("Apply CRP and check if cluster role gets propagated, update cluster role", func() { + By("create the resources to be propagated") + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-role", + Labels: map[string]string{labelKey: labelValue}, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + }, + }, + } + testutils.CreateClusterRole(ctx, *HubCluster, clusterRole) + + By("create the cluster resource placement in the hub cluster") + crp := &v1alpha1.ClusterResourcePlacement{ + ObjectMeta: v1.ObjectMeta{Name: "test-crp1"}, + Spec: v1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []v1alpha1.ClusterResourceSelector{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + LabelSelector: &v1.LabelSelector{ + MatchLabels: clusterRole.Labels, + }, + }, + }, + }, + } + + selectedResources := []v1alpha1.ResourceIdentifier{selectedResource1} + + testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) + + By("check if work gets created for cluster resource placement") + testutils.WaitWork(*HubCluster, crp.Name, memberNS.Name) + + By("check if cluster resource placement status is updated") + testutils.WaitCreateClusterResourcePlacement(ctx, *HubCluster, crp, conditionType, conditionStatus, selectedResources, targetClusters, 3*testutils.PollTimeout) + + By("check if cluster role is propagated to member cluster") + testutils.GetClusterRole(ctx, *MemberCluster, clusterRole) + + By("edit cluster role in Hub cluster") + newLabelKey := "fleet.azure.com/region" + newLabelValue := "us" + + newClusterRole := &rbacv1.ClusterRole{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-role", + Labels: map[string]string{labelKey: labelValue, newLabelKey: newLabelValue}, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + }, + }, + } + testutils.UpdateClusterRole(ctx, *HubCluster, newClusterRole) + clusterRole = &rbacv1.ClusterRole{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-role", + }, + } + + By("check if cluster role got updated in member cluster") + testutils.WaitUpdateClusterRoleLabels(ctx, *MemberCluster, clusterRole, newClusterRole) + + By("delete cluster resource placement on hub cluster") + testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) + + By("delete cluster role") + testutils.DeleteClusterRole(ctx, *HubCluster, clusterRole) + }) + + It("Apply CRP selecting namespace by label and check if namespace gets propagated with role, role binding, then update existing role", func() { + By("create the resources to be propagated") + resourceNamespace := testutils.NewNamespace("test-namespace", map[string]string{labelKey: labelValue}) + testutils.CreateNamespace(ctx, *HubCluster, resourceNamespace) + + role := &rbacv1.Role{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-pod-reader", + Namespace: "test-namespace", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Verbs: []string{"get", "list", "watch"}, + Resources: []string{"pods"}, + }, + }, + } + testutils.CreateRole(ctx, *HubCluster, role) + + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: v1.ObjectMeta{ + Name: "read-pods", + Namespace: "test-namespace", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "jane", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: "test-pod-reader", + APIGroup: "rbac.authorization.k8s.io", + }, + } + testutils.CreateRoleBinding(ctx, *HubCluster, roleBinding) + + By("create the cluster resource placement in the hub cluster") + crp := &v1alpha1.ClusterResourcePlacement{ + ObjectMeta: v1.ObjectMeta{Name: "test-crp2"}, + Spec: v1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []v1alpha1.ClusterResourceSelector{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + LabelSelector: &v1.LabelSelector{ + MatchLabels: resourceNamespace.Labels, + }, + }, + }, + }, + } + testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) + + By("check if work gets created for cluster resource placement") + testutils.WaitWork(*HubCluster, crp.Name, memberNS.Name) + + By("check if cluster resource placement status is updated") + selectedResources := []v1alpha1.ResourceIdentifier{selectedResource2, selectedResource3, selectedResource4} + testutils.WaitCreateClusterResourcePlacement(ctx, *HubCluster, crp, conditionType, conditionStatus, selectedResources, targetClusters, 3*testutils.PollTimeout) + + By("check if resources in namespace are propagated to member cluster") + testutils.GetNamespace(ctx, *MemberCluster, resourceNamespace) + testutils.GetRole(ctx, *MemberCluster, role) + testutils.GetRoleBinding(ctx, *MemberCluster, roleBinding) + + By("edit role in Hub cluster") + updatedRole := &rbacv1.Role{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-pod-reader", + Namespace: "test-namespace", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Verbs: []string{"get", "list", "watch", "update"}, + Resources: []string{"pods"}, + }, + }, + } + testutils.UpdateRole(ctx, *HubCluster, updatedRole) + role = &rbacv1.Role{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-pod-reader", + Namespace: "test-namespace", + }, + } + + By("check if role got updated in member cluster") + testutils.WaitUpdateRoleRules(ctx, *MemberCluster, role, updatedRole) + + By("check if cluster resource placement status is synced") + testutils.WaitCreateClusterResourcePlacement(ctx, *HubCluster, crp, conditionType, conditionStatus, selectedResources, targetClusters, 3*testutils.PollTimeout) + + By("delete cluster resource placement on hub cluster") + testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) + + By("delete namespace") + testutils.DeleteNamespace(ctx, *HubCluster, resourceNamespace) + }) + + It("Apply CRP selecting namespace by name and check if namespace gets propagated with role, add new role then delete new role", func() { + By("create the resources to be propagated") + resourceNamespace := testutils.NewNamespace("test-namespace", map[string]string{labelKey: labelValue}) + testutils.CreateNamespace(ctx, *HubCluster, resourceNamespace) + + role := &rbacv1.Role{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-pod-reader", + Namespace: "test-namespace", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Verbs: []string{"get", "list", "watch"}, + Resources: []string{"pods"}, + }, + }, + } + testutils.CreateRole(ctx, *HubCluster, role) + + By("create the cluster resource placement in the hub cluster") + crp := &v1alpha1.ClusterResourcePlacement{ + ObjectMeta: v1.ObjectMeta{Name: "test-crp3"}, + Spec: v1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []v1alpha1.ClusterResourceSelector{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: resourceNamespace.Name, + }, + }, + }, + } + + testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) + + By("check if work gets created for cluster resource placement") + testutils.WaitWork(*HubCluster, crp.Name, memberNS.Name) + + By("check if cluster resource placement status is updated") + var selectedResources []v1alpha1.ResourceIdentifier + selectedResources = []v1alpha1.ResourceIdentifier{selectedResource2, selectedResource4} + testutils.WaitCreateClusterResourcePlacement(ctx, *HubCluster, crp, conditionType, conditionStatus, selectedResources, targetClusters, 3*testutils.PollTimeout) + + By("check if resources in namespace are propagated to member cluster") + testutils.GetNamespace(ctx, *MemberCluster, resourceNamespace) + testutils.GetRole(ctx, *MemberCluster, role) + + By("Add new role which should be selected by cluster resource placement") + newRole := &rbacv1.Role{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-service-reader", + Namespace: "test-namespace", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Verbs: []string{"get", "list", "watch"}, + Resources: []string{"services"}, + }, + }, + } + testutils.CreateRole(ctx, *HubCluster, newRole) + + By("check if cluster resource placement status is selecting new role") + selectedResources = []v1alpha1.ResourceIdentifier{selectedResource2, selectedResource4, selectedResource5} + testutils.WaitCreateClusterResourcePlacement(ctx, *HubCluster, crp, conditionType, conditionStatus, selectedResources, targetClusters, 3*testutils.PollTimeout) + + By("check if new role in namespace is propagated to member cluster") + testutils.GetRole(ctx, *MemberCluster, newRole) + + By("delete new role in hub cluster") + testutils.DeleteRole(ctx, *HubCluster, newRole) + + By("check if role is deleted on member cluster") + testutils.WaitDeleteRole(ctx, *MemberCluster, newRole) + + By("check if cluster resource placement status is updated after deleting new role") + selectedResources = []v1alpha1.ResourceIdentifier{selectedResource2, selectedResource4} + testutils.WaitCreateClusterResourcePlacement(ctx, *HubCluster, crp, conditionType, conditionStatus, selectedResources, targetClusters, 3*testutils.PollTimeout) + + By("delete cluster resource placement on hub cluster") + testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) + + By("delete namespace") + testutils.DeleteNamespace(ctx, *HubCluster, resourceNamespace) + }) + + It("Apply CRP which selects cluster role by name and namespace by label", func() { + By("create the resources to be propagated") + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-role", + Labels: map[string]string{labelKey: labelValue}, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + }, + }, + } + testutils.CreateClusterRole(ctx, *HubCluster, clusterRole) + + resourceNamespace := testutils.NewNamespace("test-namespace", map[string]string{labelKey: labelValue}) + testutils.CreateNamespace(ctx, *HubCluster, resourceNamespace) + + role := &rbacv1.Role{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-pod-reader", + Namespace: "test-namespace", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Verbs: []string{"get", "list", "watch"}, + Resources: []string{"pods"}, + }, + }, + } + + testutils.CreateRole(ctx, *HubCluster, role) + By("create the cluster resource placement in the hub cluster") + crp := &v1alpha1.ClusterResourcePlacement{ + ObjectMeta: v1.ObjectMeta{Name: "test-crp4"}, + Spec: v1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []v1alpha1.ClusterResourceSelector{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: "test-cluster-role", + }, + { + Group: "", + Version: "v1", + Kind: "Namespace", + LabelSelector: &v1.LabelSelector{ + MatchLabels: resourceNamespace.Labels, + }, + }, + }, + }, + } + + testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) + + By("check if work gets created for cluster resource placement") + testutils.WaitWork(*HubCluster, crp.Name, memberNS.Name) + + By("check if cluster resource placement status is updated") + selectedResources := []v1alpha1.ResourceIdentifier{selectedResource1, selectedResource2, selectedResource4} + testutils.WaitCreateClusterResourcePlacement(ctx, *HubCluster, crp, conditionType, conditionStatus, selectedResources, targetClusters, 3*testutils.PollTimeout) + + By("check if cluster role, role & role binding are propagated to member cluster") + testutils.GetNamespace(ctx, *MemberCluster, resourceNamespace) + testutils.GetClusterRole(ctx, *MemberCluster, clusterRole) + testutils.GetRole(ctx, *MemberCluster, role) + + By("delete cluster resource placement on hub cluster") + testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) + + By("delete cluster role") + testutils.DeleteClusterRole(ctx, *HubCluster, clusterRole) + + By("delete namespace") + testutils.DeleteNamespace(ctx, *HubCluster, resourceNamespace) }) - It("Apply CRP and check if work gets propagated", func() { - workName := "resource-label-selector" - labelKey := "fleet.azure.com/name" - labelValue := "test" + It("Apply CRP selecting cluster role, add new cluster role with different label and update CRP to select new cluster role", func() { By("create the resources to be propagated") - cr = &rbacv1.ClusterRole{ + clusterRole := &rbacv1.ClusterRole{ ObjectMeta: v1.ObjectMeta{ Name: "test-cluster-role", Labels: map[string]string{labelKey: labelValue}, @@ -83,11 +474,11 @@ var _ = Describe("workload orchestration testing", func() { }, }, } - testutils.CreateClusterRole(*HubCluster, cr) + testutils.CreateClusterRole(ctx, *HubCluster, clusterRole) By("create the cluster resource placement in the hub cluster") - crp = &v1alpha1.ClusterResourcePlacement{ - ObjectMeta: v1.ObjectMeta{Name: "resource-label-selector"}, + crp := &v1alpha1.ClusterResourcePlacement{ + ObjectMeta: v1.ObjectMeta{Name: "test-crp5"}, Spec: v1alpha1.ClusterResourcePlacementSpec{ ResourceSelectors: []v1alpha1.ClusterResourceSelector{ { @@ -95,30 +486,85 @@ var _ = Describe("workload orchestration testing", func() { Version: "v1", Kind: "ClusterRole", LabelSelector: &v1.LabelSelector{ - MatchLabels: map[string]string{"fleet.azure.com/name": "test"}, + MatchLabels: clusterRole.Labels, }, }, }, }, } - testutils.CreateClusterResourcePlacement(*HubCluster, crp) + + testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) By("check if work gets created for cluster resource placement") - testutils.WaitWork(*HubCluster, workName, memberNS.Name) - - By("check if cluster resource placement is updated to Scheduled & Applied") - testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementConditionTypeScheduled), v1.ConditionTrue, 3*testutils.PollTimeout) - testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), v1.ConditionTrue, 3*testutils.PollTimeout) - - By("check if resource is propagated to member cluster") - testutils.WaitClusterRole(*MemberCluster, cr) - - By("delete cluster resource placement & cluster role on hub cluster") - testutils.DeleteClusterResourcePlacement(*HubCluster, crp) - Eventually(func() bool { - err := HubCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: crp.Name, Namespace: ""}, crp) - return apierrors.IsNotFound(err) - }, testutils.PollTimeout, testutils.PollInterval).Should(Equal(true)) - testutils.DeleteClusterRole(*HubCluster, cr) + testutils.WaitWork(*HubCluster, crp.Name, memberNS.Name) + + By("check if cluster resource placement status is updated") + var selectedResources []v1alpha1.ResourceIdentifier + selectedResources = []v1alpha1.ResourceIdentifier{selectedResource1} + testutils.WaitCreateClusterResourcePlacement(ctx, *HubCluster, crp, conditionType, conditionStatus, selectedResources, targetClusters, 3*testutils.PollTimeout) + + By("check if cluster role is propagated to member cluster") + testutils.GetClusterRole(ctx, *MemberCluster, clusterRole) + + By("create new cluster role") + newLabelKey := "fleet.azure.com/region" + newLabelValue := "us" + newClusterRole := &rbacv1.ClusterRole{ + ObjectMeta: v1.ObjectMeta{ + Name: "new-cluster-role", + Labels: map[string]string{newLabelKey: newLabelValue}, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{""}, + Resources: []string{"pods"}, + }, + }, + } + testutils.CreateClusterRole(ctx, *HubCluster, newClusterRole) + + By("check if cluster resource placement status is not updated") + testutils.WaitCreateClusterResourcePlacement(ctx, *HubCluster, crp, conditionType, conditionStatus, selectedResources, targetClusters, 3*testutils.PollTimeout) + + By("update cluster resource placement to select new role") + newCrp := &v1alpha1.ClusterResourcePlacement{ + ObjectMeta: v1.ObjectMeta{Name: "test-crp5"}, + Spec: v1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []v1alpha1.ClusterResourceSelector{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + LabelSelector: &v1.LabelSelector{ + MatchLabels: clusterRole.Labels, + }, + }, + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + LabelSelector: &v1.LabelSelector{ + MatchLabels: newClusterRole.Labels, + }, + }, + }, + }, + } + newCrp.SetResourceVersion(crp.GetResourceVersion()) + testutils.UpdateClusterResourcePlacement(ctx, *HubCluster, newCrp) + + By("check if cluster resource placement status is updated") + selectedResources = []v1alpha1.ResourceIdentifier{selectedResource1, selectedResource6} + testutils.WaitCreateClusterResourcePlacement(ctx, *HubCluster, newCrp, conditionType, conditionStatus, selectedResources, targetClusters, 3*testutils.PollTimeout) + + By("check if new cluster role is propagated to member cluster") + testutils.GetClusterRole(ctx, *MemberCluster, newClusterRole) + + By("delete cluster resource placement on hub cluster") + testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, newCrp) + + By("delete cluster role") + testutils.DeleteClusterRole(ctx, *HubCluster, clusterRole) }) })