From 35a30ab2d92d73e405a261ad1f873c70e642ea19 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 20 Sep 2022 15:26:47 -0700 Subject: [PATCH 01/16] Move join/leave to Before/After Suites --- test/e2e-join-leave-placement/e2e_test.go | 54 +++++ .../join_leave_workload_test.go | 209 ++++++++++++++++++ test/e2e/e2e_test.go | 191 +++++++++++++++- test/e2e/join_leave_member_test.go | 67 ------ test/e2e/utils/helper.go | 125 +---------- test/e2e/work_api_e2e_test.go | 21 +- test/e2e/work_load_test.go | 126 +---------- 7 files changed, 455 insertions(+), 338 deletions(-) create mode 100644 test/e2e-join-leave-placement/e2e_test.go create mode 100644 test/e2e-join-leave-placement/join_leave_workload_test.go delete mode 100644 test/e2e/join_leave_member_test.go diff --git a/test/e2e-join-leave-placement/e2e_test.go b/test/e2e-join-leave-placement/e2e_test.go new file mode 100644 index 000000000..a8fd309d5 --- /dev/null +++ b/test/e2e-join-leave-placement/e2e_test.go @@ -0,0 +1,54 @@ +package e2e_join_leave_placement + +import ( + "context" + "go.goms.io/fleet/test/e2e/framework" + "os" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1" + + "go.goms.io/fleet/apis/v1alpha1" +) + +var ( + scheme = runtime.NewScheme() + ctx context.Context + hubURL string + hubClusterName = "kind-hub-testing" + memberClusterName = "kind-member-testing" + HubCluster = framework.NewCluster(hubClusterName, scheme) + MemberCluster = framework.NewCluster(memberClusterName, scheme) +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(workv1alpha1.AddToScheme(scheme)) + utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) +} + +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "fleet e2e suite") +} + +var _ = BeforeSuite(func() { + kubeconfig := os.Getenv("KUBECONFIG") + Expect(kubeconfig).ShouldNot(BeEmpty(), "Failure to retrieve kubeconfig") + hubURL = os.Getenv("HUB_SERVER_URL") + Expect(hubURL).ShouldNot(BeEmpty(), "Failure to retrieve Hub URL") + + // hub setup + HubCluster.HubURL = hubURL + framework.GetClusterClient(HubCluster) + //member setup + MemberCluster.HubURL = hubURL + framework.GetClusterClient(MemberCluster) +}) diff --git a/test/e2e-join-leave-placement/join_leave_workload_test.go b/test/e2e-join-leave-placement/join_leave_workload_test.go new file mode 100644 index 000000000..923bc5160 --- /dev/null +++ b/test/e2e-join-leave-placement/join_leave_workload_test.go @@ -0,0 +1,209 @@ +package e2e_join_leave_placement + +import ( + "context" + "fmt" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "go.goms.io/fleet/apis/v1alpha1" + testutils "go.goms.io/fleet/test/e2e/utils" +) + +var _ = Describe("workload orchestration testing", func() { + var ( + mc *v1alpha1.MemberCluster + crp *v1alpha1.ClusterResourcePlacement + ctx context.Context + ) + + It("Test join and leave with CRP", func() { + ctx = context.Background() + cprName := "join-leave-test" + labelKey := "fleet.azure.com/name" + labelValue := "test" + By("create the resources to be propagated") + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test2", + Labels: map[string]string{labelKey: labelValue}, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + }, + }, + } + testutils.CreateClusterRole(*HubCluster, cr) + + By("create the cluster resource placement in the hub cluster") + crp = &v1alpha1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: cprName, + }, + Spec: v1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []v1alpha1.ClusterResourceSelector{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: cr.Labels, + }, + }, + }, + }, + } + testutils.CreateClusterResourcePlacement(*HubCluster, crp) + + By("verify the resource is not propagated to member cluster") + Consistently(func() error { + return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) + }, testutils.PollTimeout, testutils.PollInterval).ShouldNot(Succeed()) + + By("add member cluster in the hub cluster") + identity := rbacv1.Subject{ + Name: "member-agent-sa", + Kind: "ServiceAccount", + Namespace: "fleet-system", + } + mc = &v1alpha1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: MemberCluster.ClusterName, + }, + Spec: v1alpha1.MemberClusterSpec{ + Identity: identity, + State: v1alpha1.ClusterStateJoin, + HeartbeatPeriodSeconds: 60, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, mc)).Should(Succeed(), "Failed to create member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("check if member cluster condition is updated to Joined") + agentStatus := []v1alpha1.AgentStatus{ + { + Type: v1alpha1.MemberAgent, + Conditions: []metav1.Condition{ + { + Reason: "InternalMemberClusterHealthy", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentHealthy), + }, + { + Reason: "InternalMemberClusterJoined", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentJoined), + }, + }, + }, + } + mcStatus := v1alpha1.MemberClusterStatus{ + AgentStatus: agentStatus, + Conditions: []metav1.Condition{ + { + Reason: "MemberClusterReadyToJoin", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), + }, + { + Reason: "MemberClusterJoined", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ConditionTypeMemberClusterJoined), + }, + }, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"), cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{})} + statusDiff := cmp.Diff(mcStatus, mc.Status, ignoreOptions...) + if statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, mcStatus) + + By("verify that the cluster resource placement is applied") + testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) + + By("verify the resource is propagated to member cluster") + Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr)).Should(Succeed()) + + By("mark the member cluster in the hub cluster as leave") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateLeave + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("verify that member cluster is marked as notReadyToJoin") + agentStatus = []v1alpha1.AgentStatus{ + { + Type: v1alpha1.MemberAgent, + Conditions: []metav1.Condition{ + { + Reason: "InternalMemberClusterHealthy", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentHealthy), + }, + { + Reason: "InternalMemberClusterLeft", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.AgentJoined), + }, + }, + }, + } + mcStatus = v1alpha1.MemberClusterStatus{ + AgentStatus: agentStatus, + Conditions: []metav1.Condition{ + { + Reason: "MemberClusterNotReadyToJoin", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), + }, + { + Reason: "MemberClusterLeft", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.ConditionTypeMemberClusterJoined), + }, + }, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"), cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{})} + statusDiff := cmp.Diff(mcStatus, mc.Status, ignoreOptions...) + if statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, mcStatus) + + By("verify that the resource is still on the member cluster") + Consistently(func() error { + return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed()) + + By("delete the crp from the hub") + testutils.DeleteClusterResourcePlacement(*HubCluster, crp) + + By("verify that the resource is still on the member cluster") + Consistently(func() error { + return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed()) + + By("delete cluster role on hub cluster") + testutils.DeleteClusterRole(*HubCluster, cr) + }) +}) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index bf8ca16fc..01596f45c 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -5,15 +5,22 @@ Licensed under the MIT license. package e2e import ( + "context" "embed" "fmt" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" "os" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1" @@ -31,12 +38,23 @@ var ( MemberCluster = framework.NewCluster(memberClusterName, scheme) hubURL string scheme = runtime.NewScheme() + mc *v1alpha1.MemberCluster + imc *v1alpha1.InternalMemberCluster + ctx context.Context // This namespace will store Member cluster-related CRs, such as v1alpha1.MemberCluster - memberNamespace = testutils.NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName)) + memberNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName), + }, + } // This namespace in HubCluster will store v1alpha1.Work to simulate Work-related features in Hub Cluster. - workNamespace = testutils.NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName)) + workNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName), + }, + } //go:embed manifests TestManifestFiles embed.FS @@ -57,25 +75,184 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { kubeconfig := os.Getenv("KUBECONFIG") Expect(kubeconfig).ShouldNot(BeEmpty(), "Failure to retrieve kubeconfig") - hubURL = os.Getenv("HUB_SERVER_URL") Expect(hubURL).ShouldNot(BeEmpty(), "Failure to retrieve Hub URL") // hub setup HubCluster.HubURL = hubURL framework.GetClusterClient(HubCluster) - //member setup MemberCluster.HubURL = hubURL framework.GetClusterClient(MemberCluster) - testutils.CreateNamespace(*MemberCluster, memberNamespace) - testutils.CreateNamespace(*HubCluster, workNamespace) + + ctx = context.Background() + + By("deploy member cluster in the hub cluster") + identity := rbacv1.Subject{ + Name: "member-agent-sa", + Kind: "ServiceAccount", + Namespace: "fleet-system", + } + mc = &v1alpha1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: MemberCluster.ClusterName, + }, + Spec: v1alpha1.MemberClusterSpec{ + Identity: identity, + State: v1alpha1.ClusterStateJoin, + HeartbeatPeriodSeconds: 60, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, mc)).Should(Succeed(), "Failed to create member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("check if internal member cluster created in the hub cluster") + imc = &v1alpha1.InternalMemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: MemberCluster.ClusterName, + Namespace: memberNamespace.Name, + }, + } + Eventually(func() error { + return HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to be synced in %s cluster", imc.Name, HubCluster.ClusterName) + + By("check if internal member cluster status is updated to Joined") + imcStatus := v1alpha1.InternalMemberClusterStatus{ + AgentStatus: []v1alpha1.AgentStatus{ + { + Type: v1alpha1.MemberAgent, + Conditions: []metav1.Condition{ + { + Reason: "InternalMemberClusterHealthy", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentHealthy), + }, + { + Reason: "InternalMemberClusterJoined", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentJoined), + }, + }, + }, + }, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { + return err + } + ignoreOptions := []cmp.Option{cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.IgnoreFields(metav1.Condition{}, + "LastTransitionTime", "ObservedGeneration"), cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat")} + statusDiff := cmp.Diff(imcStatus, imc.Status, ignoreOptions...) + if statusDiff != "" { + return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, imcStatus) + + By("check if member cluster status is updated to Joined") + mcStatus := v1alpha1.MemberClusterStatus{ + AgentStatus: imc.Status.AgentStatus, + Conditions: []metav1.Condition{ + { + Reason: "MemberClusterReadyToJoin", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), + }, + { + Reason: "MemberClusterJoined", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ConditionTypeMemberClusterJoined), + }, + }, + ResourceUsage: imc.Status.ResourceUsage, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime")} + statusDiff := cmp.Diff(mcStatus, mc.Status, ignoreOptions...) + if statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, mcStatus) }) var _ = AfterSuite(func() { - testutils.DeleteNamespace(*MemberCluster, memberNamespace) + By("update member cluster in the hub cluster") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateLeave + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + By("check if internal member cluster status is updated to Left") + imcStatus := v1alpha1.InternalMemberClusterStatus{ + AgentStatus: []v1alpha1.AgentStatus{ + { + Type: v1alpha1.MemberAgent, + Conditions: []metav1.Condition{ + { + Reason: "InternalMemberClusterHealthy", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentHealthy), + }, + { + Reason: "InternalMemberClusterLeft", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.AgentJoined), + }, + }, + }, + }, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { + return err + } + ignoreOptions := []cmp.Option{cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.IgnoreFields(metav1.Condition{}, + "LastTransitionTime", "ObservedGeneration"), cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat")} + statusDiff := cmp.Diff(imcStatus, imc.Status, ignoreOptions...) + if statusDiff != "" { + return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, imcStatus) + + By("check if member cluster status is updated to Left") + mcStatus := v1alpha1.MemberClusterStatus{ + AgentStatus: imc.Status.AgentStatus, + Conditions: []metav1.Condition{ + { + Reason: "MemberClusterNotReadyToJoin", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), + }, + { + Reason: "MemberClusterLeft", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.ConditionTypeMemberClusterJoined), + }, + }, + ResourceUsage: imc.Status.ResourceUsage, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime")} + statusDiff := cmp.Diff(mcStatus, mc.Status, ignoreOptions...) + if statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, mcStatus) + + testutils.DeleteNamespace(*MemberCluster, memberNamespace) testutils.DeleteNamespace(*HubCluster, workNamespace) + By("delete member cluster") + testutils.DeleteMemberCluster(ctx, *HubCluster, mc) }) diff --git a/test/e2e/join_leave_member_test.go b/test/e2e/join_leave_member_test.go deleted file mode 100644 index 0f4da179a..000000000 --- a/test/e2e/join_leave_member_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright (c) Microsoft Corporation. -Licensed under the MIT license. -*/ -package e2e - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "go.goms.io/fleet/apis/v1alpha1" - testutils "go.goms.io/fleet/test/e2e/utils" -) - -var _ = Describe("Join/leave member cluster testing", func() { - var mc *v1alpha1.MemberCluster - var sa *corev1.ServiceAccount - var imc *v1alpha1.InternalMemberCluster - var ctx context.Context - - BeforeEach(func() { - ctx = context.Background() - - sa = testutils.NewServiceAccount(MemberCluster.ClusterName, memberNamespace.Name) - testutils.CreateServiceAccount(*MemberCluster, sa) - - By("deploy member cluster in the hub cluster") - mc = testutils.NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin) - testutils.CreateMemberCluster(*HubCluster, mc) - - By("check if internal member cluster created in the hub cluster") - imc = testutils.NewInternalMemberCluster(MemberCluster.ClusterName, memberNamespace.Name) - testutils.WaitInternalMemberCluster(*HubCluster, imc) - - By("check if member cluster is marked as readyToJoin") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionTrue, 3*testutils.PollTimeout) - }) - - AfterEach(func() { - testutils.DeleteMemberCluster(ctx, *HubCluster, mc) - testutils.DeleteServiceAccount(*MemberCluster, sa) - - }) - - It("Join & Leave flow is successful ", func() { - By("check if internal member cluster condition is updated to Joined") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, 3*testutils.PollTimeout) - - By("check if member cluster condition is updated to Joined") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionTrue, 3*testutils.PollTimeout) - - By("update member cluster in the hub cluster") - testutils.UpdateMemberClusterState(*HubCluster, mc, v1alpha1.ClusterStateLeave) - - By("check if internal member cluster condition is updated to Left") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionFalse, 3*testutils.PollTimeout) - - By("check if member cluster is marked as notReadyToJoin") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionFalse, 3*testutils.PollTimeout) - - By("check if member cluster condition is updated to Left") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionFalse, 3*testutils.PollTimeout) - }) -}) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index b0632fc49..9bed8a2c3 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -35,121 +35,14 @@ var ( PollTimeout = 60 * time.Second ) -// NewMemberCluster return a new member cluster. -func NewMemberCluster(name string, heartbeat int32, state v1alpha1.ClusterState) *v1alpha1.MemberCluster { - identity := rbacv1.Subject{ - Name: name, - Kind: "ServiceAccount", - Namespace: "fleet-system", - } - return &v1alpha1.MemberCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: v1alpha1.MemberClusterSpec{ - Identity: identity, - State: state, - HeartbeatPeriodSeconds: heartbeat, - }, - } -} - -// NewInternalMemberCluster returns a new internal member cluster. -func NewInternalMemberCluster(name, namespace string) *v1alpha1.InternalMemberCluster { - return &v1alpha1.InternalMemberCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } -} - -// NewServiceAccount returns a new service account. -func NewServiceAccount(name, namespace string) *corev1.ServiceAccount { - return &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } -} - -// NewNamespace returns a new namespace. -func NewNamespace(name string) *corev1.Namespace { - return &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - } -} - -// CreateMemberCluster creates MemberCluster and waits for MemberCluster to exist in the hub cluster. -func CreateMemberCluster(cluster framework.Cluster, mc *v1alpha1.MemberCluster) { - ginkgo.By(fmt.Sprintf("Creating MemberCluster(%s)", mc.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), mc) - gomega.Expect(err).Should(gomega.Succeed()) - }) - klog.Infof("Waiting for MemberCluster(%s) to be synced", mc.Name) - gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: mc.Name, Namespace: ""}, mc) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) -} - -// UpdateMemberClusterState updates MemberCluster in the hub cluster. -func UpdateMemberClusterState(cluster framework.Cluster, mc *v1alpha1.MemberCluster, state v1alpha1.ClusterState) { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: mc.Name, Namespace: ""}, mc) - gomega.Expect(err).Should(gomega.Succeed()) - mc.Spec.State = state - err = cluster.KubeClient.Update(context.TODO(), mc) - gomega.Expect(err).Should(gomega.Succeed()) -} - // DeleteMemberCluster deletes MemberCluster in the hub cluster. func DeleteMemberCluster(ctx context.Context, cluster framework.Cluster, mc *v1alpha1.MemberCluster) { - ginkgo.By(fmt.Sprintf("Deleting MemberCluster(%s)", mc.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), mc) - gomega.Expect(err).Should(gomega.Succeed()) - }) - + gomega.Expect(cluster.KubeClient.Delete(ctx, mc)).Should(gomega.Succeed(), "Failed to delete member cluster %s in %s cluster", mc.Name, cluster.ClusterName) gomega.Eventually(func() bool { return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)) }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for member cluster %s to be deleted in %s cluster", mc.Name, cluster.ClusterName) } -// WaitConditionMemberCluster waits for MemberCluster to present on th hub cluster with a specific condition. -func WaitConditionMemberCluster(cluster framework.Cluster, mc *v1alpha1.MemberCluster, conditionType v1alpha1.MemberClusterConditionType, status metav1.ConditionStatus, customTimeout time.Duration) { - klog.Infof("Waiting for MemberCluster(%s) condition(%s) status(%s) to be synced", mc.Name, conditionType, status) - gomega.Eventually(func() bool { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: mc.Name, Namespace: ""}, mc) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - cond := mc.GetCondition(string(conditionType)) - return cond != nil && cond.Status == status - }, customTimeout, PollInterval).Should(gomega.Equal(true)) -} - -// WaitInternalMemberCluster waits for InternalMemberCluster to present on th hub cluster. -func WaitInternalMemberCluster(cluster framework.Cluster, imc *v1alpha1.InternalMemberCluster) { - klog.Infof("Waiting for InternalMemberCluster(%s) to be synced in the %s cluster", imc.Name, cluster.ClusterName) - gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) -} - -// WaitConditionInternalMemberCluster waits for InternalMemberCluster to present on the hub cluster with a specific condition. -// Allowing custom timeout as for join cond it needs longer than defined PollTimeout for the member agent to finish joining. -func WaitConditionInternalMemberCluster(cluster framework.Cluster, imc *v1alpha1.InternalMemberCluster, conditionType v1alpha1.AgentConditionType, status metav1.ConditionStatus, customTimeout time.Duration) { - klog.Infof("Waiting for InternalMemberCluster(%s) condition(%s) status(%s) to be synced in the %s cluster", imc.Name, conditionType, status, cluster.ClusterName) - gomega.Eventually(func() bool { - if err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { - return false - } - cond := imc.GetConditionWithType(v1alpha1.MemberAgent, string(conditionType)) - return cond != nil && cond.Status == status - }, customTimeout, PollInterval).Should(gomega.Equal(true)) -} - // CreateClusterRole create cluster role in the hub cluster. func CreateClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { ginkgo.By(fmt.Sprintf("Creating ClusterRole (%s)", cr.Name), func() { @@ -244,22 +137,6 @@ func DeleteNamespace(cluster framework.Cluster, ns *corev1.Namespace) { }) } -// CreateServiceAccount create serviceaccount. -func CreateServiceAccount(cluster framework.Cluster, sa *corev1.ServiceAccount) { - ginkgo.By(fmt.Sprintf("Creating ServiceAccount(%s)", sa.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), sa) - gomega.Expect(err).Should(gomega.Succeed()) - }) -} - -// DeleteServiceAccount delete serviceaccount. -func DeleteServiceAccount(cluster framework.Cluster, sa *corev1.ServiceAccount) { - ginkgo.By(fmt.Sprintf("Delete ServiceAccount(%s)", sa.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), sa) - gomega.Expect(err).Should(gomega.SatisfyAny(gomega.Succeed(), &utils.NotFoundMatcher{})) - }) -} - // CreateWork creates Work object based on manifest given. func CreateWork(ctx context.Context, hubCluster framework.Cluster, workName, workNamespace string, manifests []workapi.Manifest) workapi.Work { work := workapi.Work{ diff --git a/test/e2e/work_api_e2e_test.go b/test/e2e/work_api_e2e_test.go index ca68402a9..4ae582f11 100644 --- a/test/e2e/work_api_e2e_test.go +++ b/test/e2e/work_api_e2e_test.go @@ -14,7 +14,6 @@ import ( "k8s.io/apimachinery/pkg/types" workapi "sigs.k8s.io/work-api/pkg/apis/v1alpha1" - "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/pkg/utils" testutils "go.goms.io/fleet/test/e2e/utils" ) @@ -29,10 +28,6 @@ var _ = Describe("Work API Controller test", func() { var ( ctx context.Context - - // These variables are used to join the member cluster. - mc *v1alpha1.MemberCluster - // Includes all works applied to the hub cluster. Used for garbage collection. works []workapi.Work @@ -52,25 +47,19 @@ var _ = Describe("Work API Controller test", func() { // This namespace in MemberCluster will store specified test resources created from the Work-api. resourceNamespaceName := "resource-namespace" + utils.RandStr() - resourceNamespace = testutils.NewNamespace(resourceNamespaceName) + resourceNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceNamespaceName, + }, + } testutils.CreateNamespace(*MemberCluster, resourceNamespace) - // Member Cluster must join the fleet for Work API to work. - By("deploy member cluster in the hub cluster") - mc = testutils.NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin) - testutils.CreateMemberCluster(*HubCluster, mc) - - By("check if member cluster condition is updated to Joined") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, metav1.ConditionTrue, testutils.PollTimeout) - //Empties the works since they were garbage collected earlier. works = []workapi.Work{} }) AfterEach(func() { testutils.DeleteWork(ctx, *HubCluster, works) - testutils.DeleteMemberCluster(ctx, *HubCluster, mc) - testutils.DeleteNamespace(*MemberCluster, resourceNamespace) }) diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index f28aee95e..6e983ad7c 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -9,65 +9,25 @@ import ( "context" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "go.goms.io/fleet/apis/v1alpha1" testutils "go.goms.io/fleet/test/e2e/utils" + rbacv1 "k8s.io/api/rbac/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var _ = Describe("workload orchestration testing", func() { - var mc *v1alpha1.MemberCluster - var imc *v1alpha1.InternalMemberCluster - var sa *corev1.ServiceAccount var crp *v1alpha1.ClusterResourcePlacement var ctx context.Context BeforeEach(func() { ctx = context.Background() - By("prepare resources in member cluster") - // create testing NS in member cluster - sa = testutils.NewServiceAccount(MemberCluster.ClusterName, memberNamespace.Name) - testutils.CreateServiceAccount(*MemberCluster, sa) }) AfterEach(func() { - By("delete the member cluster") - testutils.DeleteMemberCluster(ctx, *HubCluster, mc) - testutils.DeleteServiceAccount(*MemberCluster, sa) testutils.DeleteClusterResourcePlacement(*HubCluster, crp) }) Context("Test Workload Orchestration", func() { - BeforeEach(func() { - By("deploy member cluster in the hub cluster") - mc = testutils.NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin) - testutils.CreateMemberCluster(*HubCluster, mc) - - By("check if internal member cluster created in the hub cluster") - imc = testutils.NewInternalMemberCluster(MemberCluster.ClusterName, memberNamespace.Name) - testutils.WaitInternalMemberCluster(*HubCluster, imc) - - By("check if internal member cluster condition is updated to Joined") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, testutils.PollTimeout) - By("check if member cluster condition is updated to Joined") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionTrue, testutils.PollTimeout) - }) - - AfterEach(func() { - By("update member cluster in the hub cluster") - testutils.UpdateMemberClusterState(*HubCluster, mc, v1alpha1.ClusterStateLeave) - - By("check if internal member cluster condition is updated to Left") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionFalse, testutils.PollTimeout) - - By("check if member cluster is marked as notReadyToJoin") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionFalse, testutils.PollTimeout) - }) - It("Apply CRP and check if work gets propagated", func() { workName := "resource-label-selector" labelKey := "fleet.azure.com/name" @@ -120,86 +80,4 @@ var _ = Describe("workload orchestration testing", func() { testutils.DeleteClusterRole(*HubCluster, cr) }) }) - - It("Test join and leave with CRP", func() { - cprName := "join-leave-test2" - labelKey := "fleet.azure.com/name" - labelValue := "test" - By("create the resources to be propagated") - cr := &rbacv1.ClusterRole{ - ObjectMeta: v1.ObjectMeta{ - Name: "test2", - Labels: map[string]string{labelKey: labelValue}, - }, - Rules: []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list", "watch"}, - APIGroups: []string{""}, - Resources: []string{"secrets"}, - }, - }, - } - testutils.CreateClusterRole(*HubCluster, cr) - - By("create the cluster resource placement in the hub cluster") - crp = &v1alpha1.ClusterResourcePlacement{ - ObjectMeta: v1.ObjectMeta{ - Name: cprName, - }, - Spec: v1alpha1.ClusterResourcePlacementSpec{ - ResourceSelectors: []v1alpha1.ClusterResourceSelector{ - { - Group: "rbac.authorization.k8s.io", - Version: "v1", - Kind: "ClusterRole", - LabelSelector: &v1.LabelSelector{ - MatchLabels: cr.Labels, - }, - }, - }, - }, - } - testutils.CreateClusterResourcePlacement(*HubCluster, crp) - - By("verify the resource is not propagated to member cluster") - Consistently(func() error { - return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - }, testutils.PollTimeout, testutils.PollInterval).ShouldNot(Succeed()) - - By("add member cluster in the hub cluster") - mc = testutils.NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin) - testutils.CreateMemberCluster(*HubCluster, mc) - - By("check if member cluster condition is updated to Joined") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionTrue, testutils.PollTimeout) - - By("verify that the cluster resource placement is applied") - testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), v1.ConditionTrue, testutils.PollTimeout) - - By("verify the resource is propagated to member cluster") - Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr)).Should(Succeed()) - - By("mark the member cluster in the hub cluster as leave") - testutils.UpdateMemberClusterState(*HubCluster, mc, v1alpha1.ClusterStateLeave) - - By("verify that member cluster is marked as leave") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionFalse, testutils.PollTimeout) - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionFalse, testutils.PollTimeout) - - By("verify that the resource is still on the member cluster") - Consistently(func() error { - return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed()) - - By("delete the crp from the hub") - testutils.DeleteClusterResourcePlacement(*HubCluster, crp) - - By("verify that the resource is still on the member cluster") - Consistently(func() error { - return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed()) - - By("delete cluster role on hub cluster") - testutils.DeleteClusterRole(*HubCluster, cr) - }) }) From 37df92453de3385a16fb27bcc20ea26d748247b3 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 20 Sep 2022 15:31:04 -0700 Subject: [PATCH 02/16] Fix imports --- test/e2e-join-leave-placement/e2e_test.go | 2 +- test/e2e/e2e_test.go | 6 +++--- test/e2e/work_load_test.go | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/e2e-join-leave-placement/e2e_test.go b/test/e2e-join-leave-placement/e2e_test.go index a8fd309d5..9d67b0526 100644 --- a/test/e2e-join-leave-placement/e2e_test.go +++ b/test/e2e-join-leave-placement/e2e_test.go @@ -2,7 +2,6 @@ package e2e_join_leave_placement import ( "context" - "go.goms.io/fleet/test/e2e/framework" "os" "testing" @@ -15,6 +14,7 @@ import ( workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1" "go.goms.io/fleet/apis/v1alpha1" + "go.goms.io/fleet/test/e2e/framework" ) var ( diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 01596f45c..535817cc5 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -8,14 +8,14 @@ import ( "context" "embed" "fmt" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - corev1 "k8s.io/api/core/v1" "os" "testing" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index 6e983ad7c..d03c21182 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -9,10 +9,11 @@ import ( "context" . "github.com/onsi/ginkgo/v2" - "go.goms.io/fleet/apis/v1alpha1" testutils "go.goms.io/fleet/test/e2e/utils" rbacv1 "k8s.io/api/rbac/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "go.goms.io/fleet/apis/v1alpha1" ) var _ = Describe("workload orchestration testing", func() { From 9991344cb2f2d2bb34191b106d8457e8c6f7e777 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 20 Sep 2022 16:15:35 -0700 Subject: [PATCH 03/16] Minor fixes --- .../e2e_test.go | 4 +--- .../join_leave_workload_test.go | 12 ++++++------ 2 files changed, 7 insertions(+), 9 deletions(-) rename test/{e2e-join-leave-placement => e2eJoinLeavePlacement}/e2e_test.go (94%) rename test/{e2e-join-leave-placement => e2eJoinLeavePlacement}/join_leave_workload_test.go (94%) diff --git a/test/e2e-join-leave-placement/e2e_test.go b/test/e2eJoinLeavePlacement/e2e_test.go similarity index 94% rename from test/e2e-join-leave-placement/e2e_test.go rename to test/e2eJoinLeavePlacement/e2e_test.go index 9d67b0526..30d89388b 100644 --- a/test/e2e-join-leave-placement/e2e_test.go +++ b/test/e2eJoinLeavePlacement/e2e_test.go @@ -1,7 +1,6 @@ -package e2e_join_leave_placement +package e2eJoinLeavePlacement import ( - "context" "os" "testing" @@ -19,7 +18,6 @@ import ( var ( scheme = runtime.NewScheme() - ctx context.Context hubURL string hubClusterName = "kind-hub-testing" memberClusterName = "kind-member-testing" diff --git a/test/e2e-join-leave-placement/join_leave_workload_test.go b/test/e2eJoinLeavePlacement/join_leave_workload_test.go similarity index 94% rename from test/e2e-join-leave-placement/join_leave_workload_test.go rename to test/e2eJoinLeavePlacement/join_leave_workload_test.go index 923bc5160..352a36277 100644 --- a/test/e2e-join-leave-placement/join_leave_workload_test.go +++ b/test/e2eJoinLeavePlacement/join_leave_workload_test.go @@ -1,4 +1,4 @@ -package e2e_join_leave_placement +package e2eJoinLeavePlacement import ( "context" @@ -31,7 +31,7 @@ var _ = Describe("workload orchestration testing", func() { By("create the resources to be propagated") cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: "test2", + Name: "jlp-test-cluster-role", Labels: map[string]string{labelKey: labelValue}, }, Rules: []rbacv1.PolicyRule{ @@ -125,7 +125,7 @@ var _ = Describe("workload orchestration testing", func() { return err } ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), - cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"), cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{})} + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{})} statusDiff := cmp.Diff(mcStatus, mc.Status, ignoreOptions...) if statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) @@ -137,7 +137,7 @@ var _ = Describe("workload orchestration testing", func() { testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) By("verify the resource is propagated to member cluster") - Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr)).Should(Succeed()) + Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed()) By("mark the member cluster in the hub cluster as leave") Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) @@ -182,7 +182,7 @@ var _ = Describe("workload orchestration testing", func() { return err } ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), - cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"), cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{})} + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{})} statusDiff := cmp.Diff(mcStatus, mc.Status, ignoreOptions...) if statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) @@ -192,7 +192,7 @@ var _ = Describe("workload orchestration testing", func() { By("verify that the resource is still on the member cluster") Consistently(func() error { - return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) + return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed()) By("delete the crp from the hub") From 881273e20310abefe9e328e0dc98a13c83489283 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 20 Sep 2022 16:20:32 -0700 Subject: [PATCH 04/16] Makefile changes --- Makefile | 10 ++++++++-- test/e2e/work_load_test.go | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 69e5a36f9..13b5bee9a 100644 --- a/Makefile +++ b/Makefile @@ -169,11 +169,17 @@ install-member-agent-helm: install-hub-agent-helm e2e-hub-kubeconfig-secret kubectl delete pod --all -n fleet-system build-e2e: - go test -c ./test/e2e + go test -c ./test/e2e ;\ + go test -c ./test/e2e-join-leave-placement -run-e2e: build-e2e +core-e2e: KUBECONFIG=$(KUBECONFIG) HUB_SERVER_URL="https://$$(docker inspect $(HUB_KIND_CLUSTER_NAME)-control-plane --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):6443" ./e2e.test -test.v -ginkgo.v +e2e-join-leave-placement: + KUBECONFIG=$(KUBECONFIG) HUB_SERVER_URL="https://$$(docker inspect $(HUB_KIND_CLUSTER_NAME)-control-plane --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):6443" ./e2e-join-leave-placement.test -test.v -ginkgo.v + +run-e2e: build-e2e core-e2e e2e-join-leave-placement + .PHONY: creat-kind-cluster creat-kind-cluster: create-hub-kind-cluster create-member-kind-cluster install-helm diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index d03c21182..4a31ff06c 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -9,11 +9,11 @@ import ( "context" . "github.com/onsi/ginkgo/v2" - testutils "go.goms.io/fleet/test/e2e/utils" rbacv1 "k8s.io/api/rbac/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "go.goms.io/fleet/apis/v1alpha1" + testutils "go.goms.io/fleet/test/e2e/utils" ) var _ = Describe("workload orchestration testing", func() { From 4564c0b13fbabc5680b6d9758a8f99d229d4cedc Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 20 Sep 2022 16:22:36 -0700 Subject: [PATCH 05/16] Change suite name --- test/e2eJoinLeavePlacement/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2eJoinLeavePlacement/e2e_test.go b/test/e2eJoinLeavePlacement/e2e_test.go index 30d89388b..dac2f0e90 100644 --- a/test/e2eJoinLeavePlacement/e2e_test.go +++ b/test/e2eJoinLeavePlacement/e2e_test.go @@ -34,7 +34,7 @@ func init() { func TestE2E(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "fleet e2e suite") + RunSpecs(t, "fleet join leave placement e2e suite") } var _ = BeforeSuite(func() { From e608fec1b391105d02c2309194b71aa2bee77309 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 20 Sep 2022 16:40:18 -0700 Subject: [PATCH 06/16] Makefile fix --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 13b5bee9a..cbe3cf108 100644 --- a/Makefile +++ b/Makefile @@ -170,13 +170,13 @@ install-member-agent-helm: install-hub-agent-helm e2e-hub-kubeconfig-secret build-e2e: go test -c ./test/e2e ;\ - go test -c ./test/e2e-join-leave-placement + go test -c ./test/e2eJoinLeavePlacement core-e2e: KUBECONFIG=$(KUBECONFIG) HUB_SERVER_URL="https://$$(docker inspect $(HUB_KIND_CLUSTER_NAME)-control-plane --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):6443" ./e2e.test -test.v -ginkgo.v e2e-join-leave-placement: - KUBECONFIG=$(KUBECONFIG) HUB_SERVER_URL="https://$$(docker inspect $(HUB_KIND_CLUSTER_NAME)-control-plane --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):6443" ./e2e-join-leave-placement.test -test.v -ginkgo.v + KUBECONFIG=$(KUBECONFIG) HUB_SERVER_URL="https://$$(docker inspect $(HUB_KIND_CLUSTER_NAME)-control-plane --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):6443" ./e2eJoinLeavePlacement.test -test.v -ginkgo.v run-e2e: build-e2e core-e2e e2e-join-leave-placement From 0d872a04c23f471d1d4bc54826cabcbcb12ef4e4 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 20 Sep 2022 16:46:45 -0700 Subject: [PATCH 07/16] rebase fix --- test/e2eJoinLeavePlacement/join_leave_workload_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2eJoinLeavePlacement/join_leave_workload_test.go b/test/e2eJoinLeavePlacement/join_leave_workload_test.go index 352a36277..c799c1b9d 100644 --- a/test/e2eJoinLeavePlacement/join_leave_workload_test.go +++ b/test/e2eJoinLeavePlacement/join_leave_workload_test.go @@ -144,7 +144,7 @@ var _ = Describe("workload orchestration testing", func() { mc.Spec.State = v1alpha1.ClusterStateLeave Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) - By("verify that member cluster is marked as notReadyToJoin") + By("verify that member cluster is marked as left") agentStatus = []v1alpha1.AgentStatus{ { Type: v1alpha1.MemberAgent, From e2baad84eaa2c069f1e13ef42f96f16b4923f05d Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 20 Sep 2022 17:01:06 -0700 Subject: [PATCH 08/16] fix test name --- test/e2eJoinLeavePlacement/join_leave_workload_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2eJoinLeavePlacement/join_leave_workload_test.go b/test/e2eJoinLeavePlacement/join_leave_workload_test.go index c799c1b9d..4b2053d26 100644 --- a/test/e2eJoinLeavePlacement/join_leave_workload_test.go +++ b/test/e2eJoinLeavePlacement/join_leave_workload_test.go @@ -16,7 +16,7 @@ import ( testutils "go.goms.io/fleet/test/e2e/utils" ) -var _ = Describe("workload orchestration testing", func() { +var _ = Describe("workload orchestration testing with join/leave", func() { var ( mc *v1alpha1.MemberCluster crp *v1alpha1.ClusterResourcePlacement From 54fb248fe7c6db4a88b6de8a548e515d05925c4d Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 20 Sep 2022 19:11:58 -0700 Subject: [PATCH 09/16] Add fail messages to Consistently blocks --- test/e2eJoinLeavePlacement/join_leave_workload_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2eJoinLeavePlacement/join_leave_workload_test.go b/test/e2eJoinLeavePlacement/join_leave_workload_test.go index 4b2053d26..99481a0d8 100644 --- a/test/e2eJoinLeavePlacement/join_leave_workload_test.go +++ b/test/e2eJoinLeavePlacement/join_leave_workload_test.go @@ -67,7 +67,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { By("verify the resource is not propagated to member cluster") Consistently(func() error { return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) - }, testutils.PollTimeout, testutils.PollInterval).ShouldNot(Succeed()) + }, testutils.PollTimeout, testutils.PollInterval).ShouldNot(Succeed(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) By("add member cluster in the hub cluster") identity := rbacv1.Subject{ @@ -137,7 +137,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) By("verify the resource is propagated to member cluster") - Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed()) + Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName) By("mark the member cluster in the hub cluster as leave") Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) @@ -193,7 +193,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { By("verify that the resource is still on the member cluster") Consistently(func() error { return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed()) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) By("delete the crp from the hub") testutils.DeleteClusterResourcePlacement(*HubCluster, crp) @@ -201,7 +201,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { By("verify that the resource is still on the member cluster") Consistently(func() error { return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed()) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) By("delete cluster role on hub cluster") testutils.DeleteClusterRole(*HubCluster, cr) From 896827e99b69840af07bd74b79760ba7d7ff20ca Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 21 Sep 2022 11:52:48 -0700 Subject: [PATCH 10/16] Address comments --- test/e2e/e2e_test.go | 43 ++++++++----------- test/e2eJoinLeavePlacement/e2e_test.go | 2 - .../join_leave_workload_test.go | 29 ++++++------- 3 files changed, 32 insertions(+), 42 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 535817cc5..4faa78053 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -56,6 +56,13 @@ var ( }, } + sortOption = cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }) + imcStatusCmpOptions = []cmp.Option{cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), sortOption} + + mcStatusCmpOptions = []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"), sortOption} + //go:embed manifests TestManifestFiles embed.FS ) @@ -119,7 +126,7 @@ var _ = BeforeSuite(func() { }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to be synced in %s cluster", imc.Name, HubCluster.ClusterName) By("check if internal member cluster status is updated to Joined") - imcStatus := v1alpha1.InternalMemberClusterStatus{ + wantImcStatus := v1alpha1.InternalMemberClusterStatus{ AgentStatus: []v1alpha1.AgentStatus{ { Type: v1alpha1.MemberAgent, @@ -142,17 +149,14 @@ var _ = BeforeSuite(func() { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { return err } - ignoreOptions := []cmp.Option{cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.IgnoreFields(metav1.Condition{}, - "LastTransitionTime", "ObservedGeneration"), cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat")} - statusDiff := cmp.Diff(imcStatus, imc.Status, ignoreOptions...) - if statusDiff != "" { + if statusDiff := cmp.Diff(wantImcStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, imcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantImcStatus) By("check if member cluster status is updated to Joined") - mcStatus := v1alpha1.MemberClusterStatus{ + wantMcStatus := v1alpha1.MemberClusterStatus{ AgentStatus: imc.Status.AgentStatus, Conditions: []metav1.Condition{ { @@ -172,14 +176,11 @@ var _ = BeforeSuite(func() { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { return err } - ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), - cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime")} - statusDiff := cmp.Diff(mcStatus, mc.Status, ignoreOptions...) - if statusDiff != "" { + if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, mcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMcStatus) }) var _ = AfterSuite(func() { @@ -189,7 +190,7 @@ var _ = AfterSuite(func() { Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("check if internal member cluster status is updated to Left") - imcStatus := v1alpha1.InternalMemberClusterStatus{ + wantImcStatus := v1alpha1.InternalMemberClusterStatus{ AgentStatus: []v1alpha1.AgentStatus{ { Type: v1alpha1.MemberAgent, @@ -212,17 +213,14 @@ var _ = AfterSuite(func() { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { return err } - ignoreOptions := []cmp.Option{cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.IgnoreFields(metav1.Condition{}, - "LastTransitionTime", "ObservedGeneration"), cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat")} - statusDiff := cmp.Diff(imcStatus, imc.Status, ignoreOptions...) - if statusDiff != "" { + if statusDiff := cmp.Diff(wantImcStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, imcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantImcStatus) By("check if member cluster status is updated to Left") - mcStatus := v1alpha1.MemberClusterStatus{ + wantMcStatus := v1alpha1.MemberClusterStatus{ AgentStatus: imc.Status.AgentStatus, Conditions: []metav1.Condition{ { @@ -242,14 +240,11 @@ var _ = AfterSuite(func() { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { return err } - ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), - cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime")} - statusDiff := cmp.Diff(mcStatus, mc.Status, ignoreOptions...) - if statusDiff != "" { + if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, mcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMcStatus) testutils.DeleteNamespace(*MemberCluster, memberNamespace) testutils.DeleteNamespace(*HubCluster, workNamespace) diff --git a/test/e2eJoinLeavePlacement/e2e_test.go b/test/e2eJoinLeavePlacement/e2e_test.go index dac2f0e90..12fdc5ca4 100644 --- a/test/e2eJoinLeavePlacement/e2e_test.go +++ b/test/e2eJoinLeavePlacement/e2e_test.go @@ -10,7 +10,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1" "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/test/e2e/framework" @@ -28,7 +27,6 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(v1alpha1.AddToScheme(scheme)) - utilruntime.Must(workv1alpha1.AddToScheme(scheme)) utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) } diff --git a/test/e2eJoinLeavePlacement/join_leave_workload_test.go b/test/e2eJoinLeavePlacement/join_leave_workload_test.go index 99481a0d8..87df51ab8 100644 --- a/test/e2eJoinLeavePlacement/join_leave_workload_test.go +++ b/test/e2eJoinLeavePlacement/join_leave_workload_test.go @@ -21,6 +21,9 @@ var _ = Describe("workload orchestration testing with join/leave", func() { mc *v1alpha1.MemberCluster crp *v1alpha1.ClusterResourcePlacement ctx context.Context + + mcStatusCmpOptions = []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), + cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type })} ) It("Test join and leave with CRP", func() { @@ -88,7 +91,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { Expect(HubCluster.KubeClient.Create(ctx, mc)).Should(Succeed(), "Failed to create member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("check if member cluster condition is updated to Joined") - agentStatus := []v1alpha1.AgentStatus{ + mcAgentStatus := []v1alpha1.AgentStatus{ { Type: v1alpha1.MemberAgent, Conditions: []metav1.Condition{ @@ -105,8 +108,8 @@ var _ = Describe("workload orchestration testing with join/leave", func() { }, }, } - mcStatus := v1alpha1.MemberClusterStatus{ - AgentStatus: agentStatus, + wantMcStatus := v1alpha1.MemberClusterStatus{ + AgentStatus: mcAgentStatus, Conditions: []metav1.Condition{ { Reason: "MemberClusterReadyToJoin", @@ -124,14 +127,11 @@ var _ = Describe("workload orchestration testing with join/leave", func() { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { return err } - ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), - cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{})} - statusDiff := cmp.Diff(mcStatus, mc.Status, ignoreOptions...) - if statusDiff != "" { + if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, mcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMcStatus) By("verify that the cluster resource placement is applied") testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) @@ -145,7 +145,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("verify that member cluster is marked as left") - agentStatus = []v1alpha1.AgentStatus{ + mcAgentStatus = []v1alpha1.AgentStatus{ { Type: v1alpha1.MemberAgent, Conditions: []metav1.Condition{ @@ -162,8 +162,8 @@ var _ = Describe("workload orchestration testing with join/leave", func() { }, }, } - mcStatus = v1alpha1.MemberClusterStatus{ - AgentStatus: agentStatus, + wantMcStatus = v1alpha1.MemberClusterStatus{ + AgentStatus: mcAgentStatus, Conditions: []metav1.Condition{ { Reason: "MemberClusterNotReadyToJoin", @@ -181,14 +181,11 @@ var _ = Describe("workload orchestration testing with join/leave", func() { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { return err } - ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), - cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{})} - statusDiff := cmp.Diff(mcStatus, mc.Status, ignoreOptions...) - if statusDiff != "" { + if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, mcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMcStatus) By("verify that the resource is still on the member cluster") Consistently(func() error { From 0f3d801f4ae19f9003ec8955b7cef18a132f81d7 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 21 Sep 2022 21:01:42 -0700 Subject: [PATCH 11/16] Remove different suite --- Makefile | 18 +-- test/e2e/e2e_test.go | 147 ++++++++++-------- .../join_leave_workload_test.go | 144 +++++++---------- test/e2eJoinLeavePlacement/e2e_test.go | 50 ------ 4 files changed, 144 insertions(+), 215 deletions(-) rename test/{e2eJoinLeavePlacement => e2e}/join_leave_workload_test.go (61%) delete mode 100644 test/e2eJoinLeavePlacement/e2e_test.go diff --git a/Makefile b/Makefile index cbe3cf108..5c6b40b23 100644 --- a/Makefile +++ b/Makefile @@ -169,22 +169,16 @@ install-member-agent-helm: install-hub-agent-helm e2e-hub-kubeconfig-secret kubectl delete pod --all -n fleet-system build-e2e: - go test -c ./test/e2e ;\ - go test -c ./test/e2eJoinLeavePlacement + go test -c ./test/e2e -core-e2e: +run-e2e: build-e2e KUBECONFIG=$(KUBECONFIG) HUB_SERVER_URL="https://$$(docker inspect $(HUB_KIND_CLUSTER_NAME)-control-plane --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):6443" ./e2e.test -test.v -ginkgo.v -e2e-join-leave-placement: - KUBECONFIG=$(KUBECONFIG) HUB_SERVER_URL="https://$$(docker inspect $(HUB_KIND_CLUSTER_NAME)-control-plane --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):6443" ./e2eJoinLeavePlacement.test -test.v -ginkgo.v - -run-e2e: build-e2e core-e2e e2e-join-leave-placement - .PHONY: creat-kind-cluster creat-kind-cluster: create-hub-kind-cluster create-member-kind-cluster install-helm .PHONY: install-helm -install-helm: load-hub-docker-image load-member-docker-image install-member-agent-helm +install-helm: docker-build-hub-agent docker-build-member-agent docker-build-refresh-token load-hub-docker-image load-member-docker-image install-member-agent-helm .PHONY: e2e-tests e2e-tests: creat-kind-cluster run-e2e @@ -233,7 +227,7 @@ run-memberagent: manifests generate fmt vet ## Run a controllers from your host. ## Images ## -------------------------------------- -OUTPUT_TYPE ?= type=registry +OUTPUT_TYPE ?= type=docker BUILDX_BUILDER_NAME ?= img-builder QEMU_VERSION ?= 5.2.0-2 @@ -277,7 +271,7 @@ docker-build-refresh-token: docker-buildx-builder --tag $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(REFRESH_TOKEN_IMAGE_VERSION) . ## ----------------------------------- -## Cleanup +## Cleanup ## ----------------------------------- .PHONY: clean-bin @@ -305,4 +299,4 @@ clean-testing-resources: .PHONY: clean-e2e-tests clean-e2e-tests: ## Remove kind delete cluster --name $(HUB_KIND_CLUSTER_NAME) - kind delete cluster --name $(MEMBER_KIND_CLUSTER_NAME) + kind delete cluster --name $(MEMBER_KIND_CLUSTER_NAME) \ No newline at end of file diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 4faa78053..9817c5b41 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -57,11 +57,80 @@ var ( } sortOption = cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }) - imcStatusCmpOptions = []cmp.Option{cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), - cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), sortOption} + imcStatusCmpOptions = []cmp.Option{ + cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), + cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), + sortOption, + } + + mcStatusCmpOptions = []cmp.Option{ + cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), + cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"), + sortOption, + } + + imcJoinedAgentStatus = []v1alpha1.AgentStatus{ + { + Type: v1alpha1.MemberAgent, + Conditions: []metav1.Condition{ + { + Reason: "InternalMemberClusterHealthy", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentHealthy), + }, + { + Reason: "InternalMemberClusterJoined", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentJoined), + }, + }, + }, + } + imcLeftAgentStatus = []v1alpha1.AgentStatus{ + { + Type: v1alpha1.MemberAgent, + Conditions: []metav1.Condition{ + { + Reason: "InternalMemberClusterHealthy", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentHealthy), + }, + { + Reason: "InternalMemberClusterLeft", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.AgentJoined), + }, + }, + }, + } - mcStatusCmpOptions = []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), - cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"), sortOption} + mcJoinedConditions = []metav1.Condition{ + { + Reason: "MemberClusterReadyToJoin", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), + }, + { + Reason: "MemberClusterJoined", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ConditionTypeMemberClusterJoined), + }, + } + + mcLeftConditions = []metav1.Condition{ + { + Reason: "MemberClusterNotReadyToJoin", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), + }, + { + Reason: "MemberClusterLeft", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.ConditionTypeMemberClusterJoined), + }, + } //go:embed manifests TestManifestFiles embed.FS @@ -126,25 +195,7 @@ var _ = BeforeSuite(func() { }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to be synced in %s cluster", imc.Name, HubCluster.ClusterName) By("check if internal member cluster status is updated to Joined") - wantImcStatus := v1alpha1.InternalMemberClusterStatus{ - AgentStatus: []v1alpha1.AgentStatus{ - { - Type: v1alpha1.MemberAgent, - Conditions: []metav1.Condition{ - { - Reason: "InternalMemberClusterHealthy", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.AgentHealthy), - }, - { - Reason: "InternalMemberClusterJoined", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.AgentJoined), - }, - }, - }, - }, - } + wantImcStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcJoinedAgentStatus} Eventually(func() error { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { return err @@ -157,19 +208,8 @@ var _ = BeforeSuite(func() { By("check if member cluster status is updated to Joined") wantMcStatus := v1alpha1.MemberClusterStatus{ - AgentStatus: imc.Status.AgentStatus, - Conditions: []metav1.Condition{ - { - Reason: "MemberClusterReadyToJoin", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), - }, - { - Reason: "MemberClusterJoined", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.ConditionTypeMemberClusterJoined), - }, - }, + AgentStatus: imc.Status.AgentStatus, + Conditions: mcJoinedConditions, ResourceUsage: imc.Status.ResourceUsage, } Eventually(func() error { @@ -190,25 +230,7 @@ var _ = AfterSuite(func() { Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("check if internal member cluster status is updated to Left") - wantImcStatus := v1alpha1.InternalMemberClusterStatus{ - AgentStatus: []v1alpha1.AgentStatus{ - { - Type: v1alpha1.MemberAgent, - Conditions: []metav1.Condition{ - { - Reason: "InternalMemberClusterHealthy", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.AgentHealthy), - }, - { - Reason: "InternalMemberClusterLeft", - Status: metav1.ConditionFalse, - Type: string(v1alpha1.AgentJoined), - }, - }, - }, - }, - } + wantImcStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcLeftAgentStatus} Eventually(func() error { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { return err @@ -221,19 +243,8 @@ var _ = AfterSuite(func() { By("check if member cluster status is updated to Left") wantMcStatus := v1alpha1.MemberClusterStatus{ - AgentStatus: imc.Status.AgentStatus, - Conditions: []metav1.Condition{ - { - Reason: "MemberClusterNotReadyToJoin", - Status: metav1.ConditionFalse, - Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), - }, - { - Reason: "MemberClusterLeft", - Status: metav1.ConditionFalse, - Type: string(v1alpha1.ConditionTypeMemberClusterJoined), - }, - }, + AgentStatus: imc.Status.AgentStatus, + Conditions: mcLeftConditions, ResourceUsage: imc.Status.ResourceUsage, } Eventually(func() error { diff --git a/test/e2eJoinLeavePlacement/join_leave_workload_test.go b/test/e2e/join_leave_workload_test.go similarity index 61% rename from test/e2eJoinLeavePlacement/join_leave_workload_test.go rename to test/e2e/join_leave_workload_test.go index 87df51ab8..3dbdcafaf 100644 --- a/test/e2eJoinLeavePlacement/join_leave_workload_test.go +++ b/test/e2e/join_leave_workload_test.go @@ -1,4 +1,4 @@ -package e2eJoinLeavePlacement +package e2e import ( "context" @@ -16,14 +16,16 @@ import ( testutils "go.goms.io/fleet/test/e2e/utils" ) -var _ = Describe("workload orchestration testing with join/leave", func() { +var _ = Describe("workload orchestration testing with join/leave", Serial, Ordered, func() { var ( - mc *v1alpha1.MemberCluster crp *v1alpha1.ClusterResourcePlacement ctx context.Context - mcStatusCmpOptions = []cmp.Option{cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), - cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type })} + mcStatusCmpOptions = []cmp.Option{ + cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), + cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }), + } ) It("Test join and leave with CRP", func() { @@ -31,6 +33,27 @@ var _ = Describe("workload orchestration testing with join/leave", func() { cprName := "join-leave-test" labelKey := "fleet.azure.com/name" labelValue := "test" + + By("update member cluster in the hub cluster to leave") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateLeave + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("check if member cluster status is updated to Left") + wantMcStatus := v1alpha1.MemberClusterStatus{ + AgentStatus: imcLeftAgentStatus, + Conditions: mcLeftConditions, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMcStatus) + By("create the resources to be propagated") cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ @@ -72,56 +95,15 @@ var _ = Describe("workload orchestration testing with join/leave", func() { return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) }, testutils.PollTimeout, testutils.PollInterval).ShouldNot(Succeed(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) - By("add member cluster in the hub cluster") - identity := rbacv1.Subject{ - Name: "member-agent-sa", - Kind: "ServiceAccount", - Namespace: "fleet-system", - } - mc = &v1alpha1.MemberCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: MemberCluster.ClusterName, - }, - Spec: v1alpha1.MemberClusterSpec{ - Identity: identity, - State: v1alpha1.ClusterStateJoin, - HeartbeatPeriodSeconds: 60, - }, - } - Expect(HubCluster.KubeClient.Create(ctx, mc)).Should(Succeed(), "Failed to create member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + By("update member cluster in the hub cluster to join") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateJoin + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("check if member cluster condition is updated to Joined") - mcAgentStatus := []v1alpha1.AgentStatus{ - { - Type: v1alpha1.MemberAgent, - Conditions: []metav1.Condition{ - { - Reason: "InternalMemberClusterHealthy", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.AgentHealthy), - }, - { - Reason: "InternalMemberClusterJoined", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.AgentJoined), - }, - }, - }, - } - wantMcStatus := v1alpha1.MemberClusterStatus{ - AgentStatus: mcAgentStatus, - Conditions: []metav1.Condition{ - { - Reason: "MemberClusterReadyToJoin", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), - }, - { - Reason: "MemberClusterJoined", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.ConditionTypeMemberClusterJoined), - }, - }, + wantMcStatus = v1alpha1.MemberClusterStatus{ + AgentStatus: imcJoinedAgentStatus, + Conditions: mcJoinedConditions, } Eventually(func() error { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { @@ -131,7 +113,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMcStatus) By("verify that the cluster resource placement is applied") testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) @@ -139,43 +121,15 @@ var _ = Describe("workload orchestration testing with join/leave", func() { By("verify the resource is propagated to member cluster") Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName) - By("mark the member cluster in the hub cluster as leave") + By("update member cluster in the hub cluster to leave") Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) mc.Spec.State = v1alpha1.ClusterStateLeave Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("verify that member cluster is marked as left") - mcAgentStatus = []v1alpha1.AgentStatus{ - { - Type: v1alpha1.MemberAgent, - Conditions: []metav1.Condition{ - { - Reason: "InternalMemberClusterHealthy", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.AgentHealthy), - }, - { - Reason: "InternalMemberClusterLeft", - Status: metav1.ConditionFalse, - Type: string(v1alpha1.AgentJoined), - }, - }, - }, - } wantMcStatus = v1alpha1.MemberClusterStatus{ - AgentStatus: mcAgentStatus, - Conditions: []metav1.Condition{ - { - Reason: "MemberClusterNotReadyToJoin", - Status: metav1.ConditionFalse, - Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), - }, - { - Reason: "MemberClusterLeft", - Status: metav1.ConditionFalse, - Type: string(v1alpha1.ConditionTypeMemberClusterJoined), - }, - }, + AgentStatus: imcLeftAgentStatus, + Conditions: mcLeftConditions, } Eventually(func() error { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { @@ -202,5 +156,25 @@ var _ = Describe("workload orchestration testing with join/leave", func() { By("delete cluster role on hub cluster") testutils.DeleteClusterRole(*HubCluster, cr) + + By("update member cluster in the hub cluster to join") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateJoin + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("check if member cluster condition is updated to Joined") + wantMcStatus = v1alpha1.MemberClusterStatus{ + AgentStatus: imcJoinedAgentStatus, + Conditions: mcJoinedConditions, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMcStatus) }) }) diff --git a/test/e2eJoinLeavePlacement/e2e_test.go b/test/e2eJoinLeavePlacement/e2e_test.go deleted file mode 100644 index 12fdc5ca4..000000000 --- a/test/e2eJoinLeavePlacement/e2e_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package e2eJoinLeavePlacement - -import ( - "os" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - - "go.goms.io/fleet/apis/v1alpha1" - "go.goms.io/fleet/test/e2e/framework" -) - -var ( - scheme = runtime.NewScheme() - hubURL string - hubClusterName = "kind-hub-testing" - memberClusterName = "kind-member-testing" - HubCluster = framework.NewCluster(hubClusterName, scheme) - MemberCluster = framework.NewCluster(memberClusterName, scheme) -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(v1alpha1.AddToScheme(scheme)) - utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) -} - -func TestE2E(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "fleet join leave placement e2e suite") -} - -var _ = BeforeSuite(func() { - kubeconfig := os.Getenv("KUBECONFIG") - Expect(kubeconfig).ShouldNot(BeEmpty(), "Failure to retrieve kubeconfig") - hubURL = os.Getenv("HUB_SERVER_URL") - Expect(hubURL).ShouldNot(BeEmpty(), "Failure to retrieve Hub URL") - - // hub setup - HubCluster.HubURL = hubURL - framework.GetClusterClient(HubCluster) - //member setup - MemberCluster.HubURL = hubURL - framework.GetClusterClient(MemberCluster) -}) From 9cde8a46885d7de7c4d91f809ea0bf47ab886ce7 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 21 Sep 2022 21:03:50 -0700 Subject: [PATCH 12/16] Fix makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5c6b40b23..e2f977ace 100644 --- a/Makefile +++ b/Makefile @@ -178,7 +178,7 @@ run-e2e: build-e2e creat-kind-cluster: create-hub-kind-cluster create-member-kind-cluster install-helm .PHONY: install-helm -install-helm: docker-build-hub-agent docker-build-member-agent docker-build-refresh-token load-hub-docker-image load-member-docker-image install-member-agent-helm +install-helm: load-hub-docker-image load-member-docker-image install-member-agent-helm .PHONY: e2e-tests e2e-tests: creat-kind-cluster run-e2e @@ -227,7 +227,7 @@ run-memberagent: manifests generate fmt vet ## Run a controllers from your host. ## Images ## -------------------------------------- -OUTPUT_TYPE ?= type=docker +OUTPUT_TYPE ?= type=registry BUILDX_BUILDER_NAME ?= img-builder QEMU_VERSION ?= 5.2.0-2 @@ -299,4 +299,4 @@ clean-testing-resources: .PHONY: clean-e2e-tests clean-e2e-tests: ## Remove kind delete cluster --name $(HUB_KIND_CLUSTER_NAME) - kind delete cluster --name $(MEMBER_KIND_CLUSTER_NAME) \ No newline at end of file + kind delete cluster --name $(MEMBER_KIND_CLUSTER_NAME) From 2df6dc0718350636bce467b62870a85a2c5257a7 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 22 Sep 2022 09:42:59 -0700 Subject: [PATCH 13/16] Fix variable names --- test/e2e/e2e_test.go | 24 ++++++++++++------------ test/e2e/join_leave_workload_test.go | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 9817c5b41..7afa754e7 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -195,19 +195,19 @@ var _ = BeforeSuite(func() { }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to be synced in %s cluster", imc.Name, HubCluster.ClusterName) By("check if internal member cluster status is updated to Joined") - wantImcStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcJoinedAgentStatus} + wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcJoinedAgentStatus} Eventually(func() error { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { return err } - if statusDiff := cmp.Diff(wantImcStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { + if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantImcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) By("check if member cluster status is updated to Joined") - wantMcStatus := v1alpha1.MemberClusterStatus{ + wantMCStatus := v1alpha1.MemberClusterStatus{ AgentStatus: imc.Status.AgentStatus, Conditions: mcJoinedConditions, ResourceUsage: imc.Status.ResourceUsage, @@ -216,11 +216,11 @@ var _ = BeforeSuite(func() { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { return err } - if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) }) var _ = AfterSuite(func() { @@ -230,19 +230,19 @@ var _ = AfterSuite(func() { Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("check if internal member cluster status is updated to Left") - wantImcStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcLeftAgentStatus} + wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcLeftAgentStatus} Eventually(func() error { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { return err } - if statusDiff := cmp.Diff(wantImcStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { + if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantImcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) By("check if member cluster status is updated to Left") - wantMcStatus := v1alpha1.MemberClusterStatus{ + wantMCStatus := v1alpha1.MemberClusterStatus{ AgentStatus: imc.Status.AgentStatus, Conditions: mcLeftConditions, ResourceUsage: imc.Status.ResourceUsage, @@ -251,11 +251,11 @@ var _ = AfterSuite(func() { if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { return err } - if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) testutils.DeleteNamespace(*MemberCluster, memberNamespace) testutils.DeleteNamespace(*HubCluster, workNamespace) diff --git a/test/e2e/join_leave_workload_test.go b/test/e2e/join_leave_workload_test.go index 3dbdcafaf..44ce33649 100644 --- a/test/e2e/join_leave_workload_test.go +++ b/test/e2e/join_leave_workload_test.go @@ -40,7 +40,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("check if member cluster status is updated to Left") - wantMcStatus := v1alpha1.MemberClusterStatus{ + wantMCStatus := v1alpha1.MemberClusterStatus{ AgentStatus: imcLeftAgentStatus, Conditions: mcLeftConditions, } @@ -48,11 +48,11 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { return err } - if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) By("create the resources to be propagated") cr := &rbacv1.ClusterRole{ @@ -101,7 +101,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("check if member cluster condition is updated to Joined") - wantMcStatus = v1alpha1.MemberClusterStatus{ + wantMCStatus = v1alpha1.MemberClusterStatus{ AgentStatus: imcJoinedAgentStatus, Conditions: mcJoinedConditions, } @@ -109,11 +109,11 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { return err } - if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) By("verify that the cluster resource placement is applied") testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) @@ -127,7 +127,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("verify that member cluster is marked as left") - wantMcStatus = v1alpha1.MemberClusterStatus{ + wantMCStatus = v1alpha1.MemberClusterStatus{ AgentStatus: imcLeftAgentStatus, Conditions: mcLeftConditions, } @@ -135,11 +135,11 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { return err } - if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) By("verify that the resource is still on the member cluster") Consistently(func() error { @@ -163,7 +163,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) By("check if member cluster condition is updated to Joined") - wantMcStatus = v1alpha1.MemberClusterStatus{ + wantMCStatus = v1alpha1.MemberClusterStatus{ AgentStatus: imcJoinedAgentStatus, Conditions: mcJoinedConditions, } @@ -171,10 +171,10 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { return err } - if statusDiff := cmp.Diff(wantMcStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMcStatus) + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) }) }) From 49b50d954cc42328ed23bc5e548a1ac703156aa0 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 22 Sep 2022 11:26:24 -0700 Subject: [PATCH 14/16] Fix file name --- ...in_leave_workload_test.go => join_leave_member_test.go} | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) rename test/e2e/{join_leave_workload_test.go => join_leave_member_test.go} (95%) diff --git a/test/e2e/join_leave_workload_test.go b/test/e2e/join_leave_member_test.go similarity index 95% rename from test/e2e/join_leave_workload_test.go rename to test/e2e/join_leave_member_test.go index 44ce33649..810fabc91 100644 --- a/test/e2e/join_leave_workload_test.go +++ b/test/e2e/join_leave_member_test.go @@ -3,6 +3,7 @@ package e2e import ( "context" "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -91,9 +92,9 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order testutils.CreateClusterResourcePlacement(*HubCluster, crp) By("verify the resource is not propagated to member cluster") - Consistently(func() error { - return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) - }, testutils.PollTimeout, testutils.PollInterval).ShouldNot(Succeed(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) + Consistently(func() bool { + return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) By("update member cluster in the hub cluster to join") Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) From 4440a10b4c4f11c6ed049dd1b131c58cb0cc7437 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 22 Sep 2022 13:56:19 -0700 Subject: [PATCH 15/16] Address comments --- ...leave_member_test.go => join_leave_placement_test.go} | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) rename test/e2e/{join_leave_member_test.go => join_leave_placement_test.go} (95%) diff --git a/test/e2e/join_leave_member_test.go b/test/e2e/join_leave_placement_test.go similarity index 95% rename from test/e2e/join_leave_member_test.go rename to test/e2e/join_leave_placement_test.go index 810fabc91..049712f90 100644 --- a/test/e2e/join_leave_member_test.go +++ b/test/e2e/join_leave_placement_test.go @@ -1,15 +1,19 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ package e2e import ( "context" "fmt" - apierrors "k8s.io/apimachinery/pkg/api/errors" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -17,6 +21,9 @@ import ( testutils "go.goms.io/fleet/test/e2e/utils" ) +// Serial - Ginkgo will guarantee that these specs will never run in parallel with other specs. +// Ordered - Ginkgo will guarantee that specs in an Ordered container will run sequentially, in the order they are written. +// This test cannot be run in parallel with other specs in the suite as it's leaving, joining, leaving and joining again. var _ = Describe("workload orchestration testing with join/leave", Serial, Ordered, func() { var ( crp *v1alpha1.ClusterResourcePlacement From 0a99fc1f742b0645c71e54fd9b1846b9a0d294aa Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 22 Sep 2022 16:16:52 -0700 Subject: [PATCH 16/16] Removed log polltimeout --- test/e2e/join_leave_placement_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go index 049712f90..875cefaa0 100644 --- a/test/e2e/join_leave_placement_test.go +++ b/test/e2e/join_leave_placement_test.go @@ -60,7 +60,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) By("create the resources to be propagated") cr := &rbacv1.ClusterRole{ @@ -121,7 +121,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) By("verify that the cluster resource placement is applied") testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) @@ -147,7 +147,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) By("verify that the resource is still on the member cluster") Consistently(func() error { @@ -183,6 +183,6 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) } return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) }) })