test: Refactor workload orchestration E2E part 1

test/e2e/e2e_test.go

@@ -64,13 +64,16 @@ var (
 		cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"),
 		sortOption,
 	}
-
 	mcStatusCmpOptions = []cmp.Option{
 		cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"),
 		cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"),
 		cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"),
 		sortOption,
 	}
+	crpStatusCmpOptions = []cmp.Option{
+		cmpopts.IgnoreFields(metav1.Condition{}, "ObservedGeneration", "LastTransitionTime", "Message"),
+		sortOption,
+	}
 
 	imcJoinedAgentStatus = []v1alpha1.AgentStatus{
 		{
@@ -165,8 +168,6 @@ var _ = BeforeSuite(func() {
 	MemberCluster.HubURL = hubURL
 	framework.GetClusterClient(MemberCluster)
 
-	testutils.CreateNamespace(*HubCluster, workNamespace)
-
 	ctx = context.Background()
 
 	By("deploy member cluster in the hub cluster")
@@ -200,31 +201,16 @@ var _ = BeforeSuite(func() {
 
 	By("check if internal member cluster status is updated to Joined")
 	wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcJoinedAgentStatus}
-	Eventually(func() error {
-		if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil {
-			return err
-		}
-		if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" {
-			return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff)
-		}
-		return nil
-	}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus)
+	testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, wantIMCStatus, imcStatusCmpOptions)
 
 	By("check if member cluster status is updated to Joined")
+	Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc)).Should(Succeed(), "Failed to retrieve internal member cluster %s in %s cluster", imc.Name, HubCluster.ClusterName)
 	wantMCStatus := v1alpha1.MemberClusterStatus{
 		AgentStatus:   imc.Status.AgentStatus,
 		Conditions:    mcJoinedConditions,
 		ResourceUsage: imc.Status.ResourceUsage,
 	}
-	Eventually(func() error {
-		if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
-			return err
-		}
-		if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
-			return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
-		}
-		return nil
-	}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus)
+	testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)
 })
 
 var _ = AfterSuite(func() {
@@ -235,34 +221,17 @@ var _ = AfterSuite(func() {
 
 	By("check if internal member cluster status is updated to Left")
 	wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcLeftAgentStatus}
-	Eventually(func() error {
-		if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil {
-			return err
-		}
-		if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" {
-			return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff)
-		}
-		return nil
-	}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus)
+	testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, wantIMCStatus, imcStatusCmpOptions)
 
 	By("check if member cluster status is updated to Left")
+	Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc)).Should(Succeed(), "Failed to retrieve internal member cluster %s in %s cluster", imc.Name, HubCluster.ClusterName)
 	wantMCStatus := v1alpha1.MemberClusterStatus{
 		AgentStatus:   imc.Status.AgentStatus,
 		Conditions:    mcLeftConditions,
 		ResourceUsage: imc.Status.ResourceUsage,
 	}
-	Eventually(func() error {
-		if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
-			return err
-		}
-		if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
-			return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
-		}
-		return nil
-	}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus)
+	testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)
 
-	testutils.DeleteNamespace(*MemberCluster, memberNamespace)
-	testutils.DeleteNamespace(*HubCluster, workNamespace)
 	By("delete member cluster")
 	testutils.DeleteMemberCluster(ctx, *HubCluster, mc)
 })

test/e2e/join_leave_placement_test.go

@@ -6,7 +6,6 @@ package e2e
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -18,21 +17,21 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 
 	"go.goms.io/fleet/apis/v1alpha1"
-	testutils "go.goms.io/fleet/test/e2e/utils"
+	"go.goms.io/fleet/test/e2e/utils"
 )
 
 // Serial - Ginkgo will guarantee that these specs will never run in parallel with other specs.
-// Ordered - Ginkgo will guarantee that specs in an Ordered container will run sequentially, in the order they are written.
 // This test cannot be run in parallel with other specs in the suite as it's leaving, joining, leaving and joining again.
-var _ = Describe("workload orchestration testing with join/leave", Serial, Ordered, func() {
+var _ = Describe("workload orchestration testing with join/leave", Serial, func() {
 	var (
 		crp *v1alpha1.ClusterResourcePlacement
 		ctx context.Context
 
 		mcStatusCmpOptions = []cmp.Option{
 			cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"),
 			cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"),
-			cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }),
+			cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}),
+			sortOption,
 		}
 	)
 
@@ -52,15 +51,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
 			AgentStatus: imcLeftAgentStatus,
 			Conditions:  mcLeftConditions,
 		}
-		Eventually(func() error {
-			if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
-				return err
-			}
-			if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
-				return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
-			}
-			return nil
-		}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus)
+		utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)
 
 		By("create the resources to be propagated")
 		cr := &rbacv1.ClusterRole{
@@ -76,7 +67,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
 				},
 			},
 		}
-		testutils.CreateClusterRole(*HubCluster, cr)
+		Expect(HubCluster.KubeClient.Create(ctx, cr)).Should(Succeed(), "Failed to create cluster role %s in %s cluster", cr.Name, HubCluster.ClusterName)
 
 		By("create the cluster resource placement in the hub cluster")
 		crp = &v1alpha1.ClusterResourcePlacement{
@@ -96,12 +87,12 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
 				},
 			},
 		}
-		testutils.CreateClusterResourcePlacement(*HubCluster, crp)
+		Expect(HubCluster.KubeClient.Create(ctx, crp)).Should(Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, HubCluster.ClusterName)
 
 		By("verify the resource is not propagated to member cluster")
 		Consistently(func() bool {
 			return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr))
-		}, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName)
+		}, utils.PollTimeout, utils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName)
 
 		By("update member cluster in the hub cluster to join")
 		Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName)
@@ -113,18 +104,33 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
 			AgentStatus: imcJoinedAgentStatus,
 			Conditions:  mcJoinedConditions,
 		}
-		Eventually(func() error {
-			if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
-				return err
-			}
-			if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
-				return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
-			}
-			return nil
-		}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus)
+		utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)
 
 		By("verify that the cluster resource placement is applied")
-		testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout)
+		crpStatus := v1alpha1.ClusterResourcePlacementStatus{
+			Conditions: []metav1.Condition{
+				{
+					Reason: "ScheduleSucceeded",
+					Status: metav1.ConditionTrue,
+					Type:   string(v1alpha1.ResourcePlacementConditionTypeScheduled),
+				},
+				{
+					Reason: "ApplySucceeded",
+					Status: metav1.ConditionTrue,
+					Type:   string(v1alpha1.ResourcePlacementStatusConditionTypeApplied),
+				},
+			},
+			SelectedResources: []v1alpha1.ResourceIdentifier{
+				{
+					Group:   "rbac.authorization.k8s.io",
+					Version: "v1",
+					Kind:    "ClusterRole",
+					Name:    cr.Name,
+				},
+			},
+			TargetClusters: []string{"kind-member-testing"},
+		}
+		utils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*utils.PollTimeout)
 
 		By("verify the resource is propagated to member cluster")
 		Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName)
@@ -139,31 +145,26 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
 			AgentStatus: imcLeftAgentStatus,
 			Conditions:  mcLeftConditions,
 		}
-		Eventually(func() error {
-			if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
-				return err
-			}
-			if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
-				return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
-			}
-			return nil
-		}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus)
+		utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)
 
 		By("verify that the resource is still on the member cluster")
 		Consistently(func() error {
 			return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)
-		}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)
+		}, utils.PollTimeout, utils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)
 
 		By("delete the crp from the hub")
-		testutils.DeleteClusterResourcePlacement(*HubCluster, crp)
+		utils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp)
 
 		By("verify that the resource is still on the member cluster")
 		Consistently(func() error {
 			return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr)
-		}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)
+		}, utils.PollTimeout, utils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)
 
 		By("delete cluster role on hub cluster")
-		testutils.DeleteClusterRole(*HubCluster, cr)
+		Expect(HubCluster.KubeClient.Delete(ctx, cr)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", cr.Name, HubCluster.ClusterName)
+
+		By("delete cluster role on member cluster")
+		Expect(MemberCluster.KubeClient.Delete(ctx, cr)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", cr.Name, MemberCluster.ClusterName)
 
 		By("update member cluster in the hub cluster to join")
 		Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName)
@@ -175,14 +176,6 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
 			AgentStatus: imcJoinedAgentStatus,
 			Conditions:  mcJoinedConditions,
 		}
-		Eventually(func() error {
-			if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
-				return err
-			}
-			if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
-				return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
-			}
-			return nil
-		}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus)
+		utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)
 	})
 })