From c8f8c500d1a3cf8c3efba9f5bfe1d42303599db2 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 27 Sep 2022 06:31:00 +0530 Subject: [PATCH 1/5] New Suite --- go.mod | 3 + go.sum | 5 + test/e2e/join_leave_placement_test.go | 365 +++++++++--------- test/e2eJoinLeavePlacement/e2e_test.go | 118 ++++++ .../join_leave_placement_test.go | 74 ++++ 5 files changed, 383 insertions(+), 182 deletions(-) create mode 100644 test/e2eJoinLeavePlacement/e2e_test.go create mode 100644 test/e2eJoinLeavePlacement/join_leave_placement_test.go diff --git a/go.mod b/go.mod index 2421935e4..f5877795a 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/crossplane/crossplane-runtime v0.16.0 github.com/go-logr/logr v1.2.3 github.com/google/go-cmp v0.5.8 + github.com/onsi/ginkgo v1.16.5 github.com/onsi/ginkgo/v2 v2.1.6 github.com/onsi/gomega v1.20.1 github.com/openkruise/kruise v1.2.0 @@ -64,6 +65,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nxadm/tail v1.4.8 // indirect github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -81,6 +83,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect diff --git a/go.sum b/go.sum index 6e7c3cfdb..3c3de0102 100644 --- a/go.sum +++ b/go.sum @@ -181,6 +181,7 @@ github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/ github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -378,12 +379,14 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -695,6 +698,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -788,6 +792,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go index 875cefaa0..b1f71dc57 100644 --- a/test/e2e/join_leave_placement_test.go +++ b/test/e2e/join_leave_placement_test.go @@ -4,185 +4,186 @@ Licensed under the MIT license. */ package e2e -import ( - "context" - "fmt" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - "go.goms.io/fleet/apis/v1alpha1" - testutils "go.goms.io/fleet/test/e2e/utils" -) - -// Serial - Ginkgo will guarantee that these specs will never run in parallel with other specs. -// Ordered - Ginkgo will guarantee that specs in an Ordered container will run sequentially, in the order they are written. -// This test cannot be run in parallel with other specs in the suite as it's leaving, joining, leaving and joining again. -var _ = Describe("workload orchestration testing with join/leave", Serial, Ordered, func() { - var ( - crp *v1alpha1.ClusterResourcePlacement - ctx context.Context - - mcStatusCmpOptions = []cmp.Option{ - cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), - cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), - cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }), - } - ) - - It("Test join and leave with CRP", func() { - ctx = context.Background() - cprName := "join-leave-test" - labelKey := "fleet.azure.com/name" - labelValue := "test" - - By("update member cluster in the hub cluster to leave") - Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) - mc.Spec.State = v1alpha1.ClusterStateLeave - Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) - - By("check if member cluster status is updated to Left") - wantMCStatus := v1alpha1.MemberClusterStatus{ - AgentStatus: imcLeftAgentStatus, - Conditions: mcLeftConditions, - } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) - - By("create the resources to be propagated") - cr := &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: "jlp-test-cluster-role", - Labels: map[string]string{labelKey: labelValue}, - }, - Rules: []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list", "watch"}, - APIGroups: []string{""}, - Resources: []string{"secrets"}, - }, - }, - } - testutils.CreateClusterRole(*HubCluster, cr) - - By("create the cluster resource placement in the hub cluster") - crp = &v1alpha1.ClusterResourcePlacement{ - ObjectMeta: metav1.ObjectMeta{ - Name: cprName, - }, - Spec: v1alpha1.ClusterResourcePlacementSpec{ - ResourceSelectors: []v1alpha1.ClusterResourceSelector{ - { - Group: "rbac.authorization.k8s.io", - Version: "v1", - Kind: "ClusterRole", - LabelSelector: &metav1.LabelSelector{ - MatchLabels: cr.Labels, - }, - }, - }, - }, - } - testutils.CreateClusterResourcePlacement(*HubCluster, crp) - - By("verify the resource is not propagated to member cluster") - Consistently(func() bool { - return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)) - }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) - - By("update member cluster in the hub cluster to join") - Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) - mc.Spec.State = v1alpha1.ClusterStateJoin - Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) - - By("check if member cluster condition is updated to Joined") - wantMCStatus = v1alpha1.MemberClusterStatus{ - AgentStatus: imcJoinedAgentStatus, - Conditions: mcJoinedConditions, - } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) - - By("verify that the cluster resource placement is applied") - testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) - - By("verify the resource is propagated to member cluster") - Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName) - - By("update member cluster in the hub cluster to leave") - Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) - mc.Spec.State = v1alpha1.ClusterStateLeave - Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) - - By("verify that member cluster is marked as left") - wantMCStatus = v1alpha1.MemberClusterStatus{ - AgentStatus: imcLeftAgentStatus, - Conditions: mcLeftConditions, - } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) - - By("verify that the resource is still on the member cluster") - Consistently(func() error { - return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) - - By("delete the crp from the hub") - testutils.DeleteClusterResourcePlacement(*HubCluster, crp) - - By("verify that the resource is still on the member cluster") - Consistently(func() error { - return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) - - By("delete cluster role on hub cluster") - testutils.DeleteClusterRole(*HubCluster, cr) - - By("update member cluster in the hub cluster to join") - Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) - mc.Spec.State = v1alpha1.ClusterStateJoin - Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) - - By("check if member cluster condition is updated to Joined") - wantMCStatus = v1alpha1.MemberClusterStatus{ - AgentStatus: imcJoinedAgentStatus, - Conditions: mcJoinedConditions, - } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) - }) -}) +// +//import ( +// "context" +// "fmt" +// +// "github.com/google/go-cmp/cmp" +// "github.com/google/go-cmp/cmp/cmpopts" +// . "github.com/onsi/ginkgo/v2" +// . "github.com/onsi/gomega" +// rbacv1 "k8s.io/api/rbac/v1" +// apierrors "k8s.io/apimachinery/pkg/api/errors" +// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// "k8s.io/apimachinery/pkg/types" +// +// "go.goms.io/fleet/apis/v1alpha1" +// testutils "go.goms.io/fleet/test/e2e/utils" +//) +// +//// Serial - Ginkgo will guarantee that these specs will never run in parallel with other specs. +//// Ordered - Ginkgo will guarantee that specs in an Ordered container will run sequentially, in the order they are written. +//// This test cannot be run in parallel with other specs in the suite as it's leaving, joining, leaving and joining again. +//var _ = Describe("workload orchestration testing with join/leave", Serial, Ordered, func() { +// var ( +// crp *v1alpha1.ClusterResourcePlacement +// ctx context.Context +// +// mcStatusCmpOptions = []cmp.Option{ +// cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), +// cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), +// cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }), +// } +// ) +// +// It("Test join and leave with CRP", func() { +// ctx = context.Background() +// cprName := "join-leave-test" +// labelKey := "fleet.azure.com/name" +// labelValue := "test" +// +// By("update member cluster in the hub cluster to leave") +// Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) +// mc.Spec.State = v1alpha1.ClusterStateLeave +// Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) +// +// By("check if member cluster status is updated to Left") +// wantMCStatus := v1alpha1.MemberClusterStatus{ +// AgentStatus: imcLeftAgentStatus, +// Conditions: mcLeftConditions, +// } +// Eventually(func() error { +// if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { +// return err +// } +// if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { +// return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) +// } +// return nil +// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) +// +// By("create the resources to be propagated") +// cr := &rbacv1.ClusterRole{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "jlp-test-cluster-role", +// Labels: map[string]string{labelKey: labelValue}, +// }, +// Rules: []rbacv1.PolicyRule{ +// { +// Verbs: []string{"get", "list", "watch"}, +// APIGroups: []string{""}, +// Resources: []string{"secrets"}, +// }, +// }, +// } +// testutils.CreateClusterRole(*HubCluster, cr) +// +// By("create the cluster resource placement in the hub cluster") +// crp = &v1alpha1.ClusterResourcePlacement{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: cprName, +// }, +// Spec: v1alpha1.ClusterResourcePlacementSpec{ +// ResourceSelectors: []v1alpha1.ClusterResourceSelector{ +// { +// Group: "rbac.authorization.k8s.io", +// Version: "v1", +// Kind: "ClusterRole", +// LabelSelector: &metav1.LabelSelector{ +// MatchLabels: cr.Labels, +// }, +// }, +// }, +// }, +// } +// testutils.CreateClusterResourcePlacement(*HubCluster, crp) +// +// By("verify the resource is not propagated to member cluster") +// Consistently(func() bool { +// return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)) +// }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) +// +// By("update member cluster in the hub cluster to join") +// Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) +// mc.Spec.State = v1alpha1.ClusterStateJoin +// Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) +// +// By("check if member cluster condition is updated to Joined") +// wantMCStatus = v1alpha1.MemberClusterStatus{ +// AgentStatus: imcJoinedAgentStatus, +// Conditions: mcJoinedConditions, +// } +// Eventually(func() error { +// if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { +// return err +// } +// if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { +// return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) +// } +// return nil +// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) +// +// By("verify that the cluster resource placement is applied") +// testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) +// +// By("verify the resource is propagated to member cluster") +// Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName) +// +// By("update member cluster in the hub cluster to leave") +// Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) +// mc.Spec.State = v1alpha1.ClusterStateLeave +// Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) +// +// By("verify that member cluster is marked as left") +// wantMCStatus = v1alpha1.MemberClusterStatus{ +// AgentStatus: imcLeftAgentStatus, +// Conditions: mcLeftConditions, +// } +// Eventually(func() error { +// if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { +// return err +// } +// if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { +// return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) +// } +// return nil +// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) +// +// By("verify that the resource is still on the member cluster") +// Consistently(func() error { +// return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) +// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) +// +// By("delete the crp from the hub") +// testutils.DeleteClusterResourcePlacement(*HubCluster, crp) +// +// By("verify that the resource is still on the member cluster") +// Consistently(func() error { +// return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) +// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) +// +// By("delete cluster role on hub cluster") +// testutils.DeleteClusterRole(*HubCluster, cr) +// +// By("update member cluster in the hub cluster to join") +// Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) +// mc.Spec.State = v1alpha1.ClusterStateJoin +// Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) +// +// By("check if member cluster condition is updated to Joined") +// wantMCStatus = v1alpha1.MemberClusterStatus{ +// AgentStatus: imcJoinedAgentStatus, +// Conditions: mcJoinedConditions, +// } +// Eventually(func() error { +// if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { +// return err +// } +// if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { +// return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) +// } +// return nil +// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) +// }) +//}) diff --git a/test/e2eJoinLeavePlacement/e2e_test.go b/test/e2eJoinLeavePlacement/e2e_test.go new file mode 100644 index 000000000..369fc98b1 --- /dev/null +++ b/test/e2eJoinLeavePlacement/e2e_test.go @@ -0,0 +1,118 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package e2eJoinLeavePlacement + +import ( + "context" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "os" + "testing" + + "go.goms.io/fleet/apis/v1alpha1" + "go.goms.io/fleet/test/e2e/framework" +) + +var ( + hubClusterName = "kind-hub-testing" + memberClusterName = "kind-member-testing" + HubCluster = framework.NewCluster(hubClusterName, scheme) + MemberCluster = framework.NewCluster(memberClusterName, scheme) + hubURL string + scheme = runtime.NewScheme() + ctx context.Context + + imcJoinedAgentStatus = []v1alpha1.AgentStatus{ + { + Type: v1alpha1.MemberAgent, + Conditions: []metav1.Condition{ + { + Reason: "InternalMemberClusterHealthy", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentHealthy), + }, + { + Reason: "InternalMemberClusterJoined", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentJoined), + }, + }, + }, + } + imcLeftAgentStatus = []v1alpha1.AgentStatus{ + { + Type: v1alpha1.MemberAgent, + Conditions: []metav1.Condition{ + { + Reason: "InternalMemberClusterHealthy", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentHealthy), + }, + { + Reason: "InternalMemberClusterLeft", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.AgentJoined), + }, + }, + }, + } + + mcJoinedConditions = []metav1.Condition{ + { + Reason: "MemberClusterReadyToJoin", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), + }, + { + Reason: "MemberClusterJoined", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ConditionTypeMemberClusterJoined), + }, + } + + mcLeftConditions = []metav1.Condition{ + { + Reason: "MemberClusterNotReadyToJoin", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), + }, + { + Reason: "MemberClusterLeft", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.ConditionTypeMemberClusterJoined), + }, + } +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) +} + +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "fleet e2e workload suite") +} + +var _ = BeforeSuite(func() { + kubeconfig := os.Getenv("KUBECONFIG") + Expect(kubeconfig).ShouldNot(BeEmpty(), "Failure to retrieve kubeconfig") + hubURL = os.Getenv("HUB_SERVER_URL") + Expect(hubURL).ShouldNot(BeEmpty(), "Failure to retrieve Hub URL") + + // hub setup + HubCluster.HubURL = hubURL + framework.GetClusterClient(HubCluster) + //member setup + MemberCluster.HubURL = hubURL + framework.GetClusterClient(MemberCluster) +}) diff --git a/test/e2eJoinLeavePlacement/join_leave_placement_test.go b/test/e2eJoinLeavePlacement/join_leave_placement_test.go new file mode 100644 index 000000000..80341511d --- /dev/null +++ b/test/e2eJoinLeavePlacement/join_leave_placement_test.go @@ -0,0 +1,74 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package e2eJoinLeavePlacement + +import ( + "context" + "fmt" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + testutils "go.goms.io/fleet/test/e2e/utils" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "go.goms.io/fleet/apis/v1alpha1" +) + +var ( + mc *v1alpha1.MemberCluster + mcStatusCmpOptions = []cmp.Option{ + cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), + cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }), + } +) + +var _ = Describe("workload orchestration testing with join/leave", func() { + + It("Test join and leave with CRP", func() { + ctx = context.Background() + //cprName := "join-leave-test" + //labelKey := "fleet.azure.com/name" + //labelValue := "test" + + By("deploy member cluster in the hub cluster") + identity := rbacv1.Subject{ + Name: "member-agent-sa", + Kind: "ServiceAccount", + Namespace: "fleet-system", + } + mc = &v1alpha1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: MemberCluster.ClusterName, + }, + Spec: v1alpha1.MemberClusterSpec{ + Identity: identity, + State: v1alpha1.ClusterStateJoin, + HeartbeatPeriodSeconds: 60, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, mc)).Should(Succeed(), "Failed to create member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("check if member cluster status is updated to joined") + wantMCStatus := v1alpha1.MemberClusterStatus{ + AgentStatus: imcJoinedAgentStatus, + Conditions: mcJoinedConditions, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + }) +}) From d6c0ad3c2d7c2337aca2f365a4bc6d779cad36d4 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 27 Sep 2022 06:34:55 +0530 Subject: [PATCH 2/5] Working e2e suite --- test/e2eJoinLeavePlacement/e2e_test.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/test/e2eJoinLeavePlacement/e2e_test.go b/test/e2eJoinLeavePlacement/e2e_test.go index 369fc98b1..03d9a2074 100644 --- a/test/e2eJoinLeavePlacement/e2e_test.go +++ b/test/e2eJoinLeavePlacement/e2e_test.go @@ -1,34 +1,31 @@ -/* -Copyright (c) Microsoft Corporation. -Licensed under the MIT license. -*/ - package e2eJoinLeavePlacement import ( "context" - . "github.com/onsi/ginkgo" + "os" + "testing" + + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "os" - "testing" + workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1" "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/test/e2e/framework" ) var ( + ctx context.Context + scheme = runtime.NewScheme() + hubURL string hubClusterName = "kind-hub-testing" memberClusterName = "kind-member-testing" HubCluster = framework.NewCluster(hubClusterName, scheme) MemberCluster = framework.NewCluster(memberClusterName, scheme) - hubURL string - scheme = runtime.NewScheme() - ctx context.Context imcJoinedAgentStatus = []v1alpha1.AgentStatus{ { @@ -95,15 +92,17 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(workv1alpha1.AddToScheme(scheme)) utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) } func TestE2E(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "fleet e2e workload suite") + RunSpecs(t, "fleet join leave placement e2e suite") } var _ = BeforeSuite(func() { + ctx = context.Background() kubeconfig := os.Getenv("KUBECONFIG") Expect(kubeconfig).ShouldNot(BeEmpty(), "Failure to retrieve kubeconfig") hubURL = os.Getenv("HUB_SERVER_URL") From f708a0263c9e9343124dc0d2f69a844c2c277c0a Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 27 Sep 2022 07:32:57 +0530 Subject: [PATCH 3/5] Makefile changes --- Makefile | 10 +- test/e2e/join_leave_placement_test.go | 189 ------------------ test/e2eJoinLeavePlacement/e2e_test.go | 6 +- .../join_leave_placement_test.go | 97 ++++++++- 4 files changed, 105 insertions(+), 197 deletions(-) delete mode 100644 test/e2e/join_leave_placement_test.go diff --git a/Makefile b/Makefile index e2f977ace..504631e92 100644 --- a/Makefile +++ b/Makefile @@ -169,11 +169,17 @@ install-member-agent-helm: install-hub-agent-helm e2e-hub-kubeconfig-secret kubectl delete pod --all -n fleet-system build-e2e: - go test -c ./test/e2e + go test -c ./test/e2e ;\ + go test -c ./test/e2eJoinLeavePlacement -run-e2e: build-e2e +core-e2e: KUBECONFIG=$(KUBECONFIG) HUB_SERVER_URL="https://$$(docker inspect $(HUB_KIND_CLUSTER_NAME)-control-plane --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):6443" ./e2e.test -test.v -ginkgo.v +e2e-join-leave-placement: + KUBECONFIG=$(KUBECONFIG) HUB_SERVER_URL="https://$$(docker inspect $(HUB_KIND_CLUSTER_NAME)-control-plane --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):6443" ./e2eJoinLeavePlacement.test -test.v -ginkgo.v + +run-e2e: build-e2e core-e2e e2e-join-leave-placement + .PHONY: creat-kind-cluster creat-kind-cluster: create-hub-kind-cluster create-member-kind-cluster install-helm diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go deleted file mode 100644 index b1f71dc57..000000000 --- a/test/e2e/join_leave_placement_test.go +++ /dev/null @@ -1,189 +0,0 @@ -/* -Copyright (c) Microsoft Corporation. -Licensed under the MIT license. -*/ -package e2e - -// -//import ( -// "context" -// "fmt" -// -// "github.com/google/go-cmp/cmp" -// "github.com/google/go-cmp/cmp/cmpopts" -// . "github.com/onsi/ginkgo/v2" -// . "github.com/onsi/gomega" -// rbacv1 "k8s.io/api/rbac/v1" -// apierrors "k8s.io/apimachinery/pkg/api/errors" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// "k8s.io/apimachinery/pkg/types" -// -// "go.goms.io/fleet/apis/v1alpha1" -// testutils "go.goms.io/fleet/test/e2e/utils" -//) -// -//// Serial - Ginkgo will guarantee that these specs will never run in parallel with other specs. -//// Ordered - Ginkgo will guarantee that specs in an Ordered container will run sequentially, in the order they are written. -//// This test cannot be run in parallel with other specs in the suite as it's leaving, joining, leaving and joining again. -//var _ = Describe("workload orchestration testing with join/leave", Serial, Ordered, func() { -// var ( -// crp *v1alpha1.ClusterResourcePlacement -// ctx context.Context -// -// mcStatusCmpOptions = []cmp.Option{ -// cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), -// cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), -// cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }), -// } -// ) -// -// It("Test join and leave with CRP", func() { -// ctx = context.Background() -// cprName := "join-leave-test" -// labelKey := "fleet.azure.com/name" -// labelValue := "test" -// -// By("update member cluster in the hub cluster to leave") -// Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) -// mc.Spec.State = v1alpha1.ClusterStateLeave -// Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) -// -// By("check if member cluster status is updated to Left") -// wantMCStatus := v1alpha1.MemberClusterStatus{ -// AgentStatus: imcLeftAgentStatus, -// Conditions: mcLeftConditions, -// } -// Eventually(func() error { -// if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { -// return err -// } -// if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { -// return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) -// } -// return nil -// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) -// -// By("create the resources to be propagated") -// cr := &rbacv1.ClusterRole{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: "jlp-test-cluster-role", -// Labels: map[string]string{labelKey: labelValue}, -// }, -// Rules: []rbacv1.PolicyRule{ -// { -// Verbs: []string{"get", "list", "watch"}, -// APIGroups: []string{""}, -// Resources: []string{"secrets"}, -// }, -// }, -// } -// testutils.CreateClusterRole(*HubCluster, cr) -// -// By("create the cluster resource placement in the hub cluster") -// crp = &v1alpha1.ClusterResourcePlacement{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: cprName, -// }, -// Spec: v1alpha1.ClusterResourcePlacementSpec{ -// ResourceSelectors: []v1alpha1.ClusterResourceSelector{ -// { -// Group: "rbac.authorization.k8s.io", -// Version: "v1", -// Kind: "ClusterRole", -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: cr.Labels, -// }, -// }, -// }, -// }, -// } -// testutils.CreateClusterResourcePlacement(*HubCluster, crp) -// -// By("verify the resource is not propagated to member cluster") -// Consistently(func() bool { -// return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)) -// }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) -// -// By("update member cluster in the hub cluster to join") -// Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) -// mc.Spec.State = v1alpha1.ClusterStateJoin -// Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) -// -// By("check if member cluster condition is updated to Joined") -// wantMCStatus = v1alpha1.MemberClusterStatus{ -// AgentStatus: imcJoinedAgentStatus, -// Conditions: mcJoinedConditions, -// } -// Eventually(func() error { -// if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { -// return err -// } -// if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { -// return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) -// } -// return nil -// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) -// -// By("verify that the cluster resource placement is applied") -// testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) -// -// By("verify the resource is propagated to member cluster") -// Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName) -// -// By("update member cluster in the hub cluster to leave") -// Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) -// mc.Spec.State = v1alpha1.ClusterStateLeave -// Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) -// -// By("verify that member cluster is marked as left") -// wantMCStatus = v1alpha1.MemberClusterStatus{ -// AgentStatus: imcLeftAgentStatus, -// Conditions: mcLeftConditions, -// } -// Eventually(func() error { -// if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { -// return err -// } -// if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { -// return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) -// } -// return nil -// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) -// -// By("verify that the resource is still on the member cluster") -// Consistently(func() error { -// return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) -// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) -// -// By("delete the crp from the hub") -// testutils.DeleteClusterResourcePlacement(*HubCluster, crp) -// -// By("verify that the resource is still on the member cluster") -// Consistently(func() error { -// return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) -// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) -// -// By("delete cluster role on hub cluster") -// testutils.DeleteClusterRole(*HubCluster, cr) -// -// By("update member cluster in the hub cluster to join") -// Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) -// mc.Spec.State = v1alpha1.ClusterStateJoin -// Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) -// -// By("check if member cluster condition is updated to Joined") -// wantMCStatus = v1alpha1.MemberClusterStatus{ -// AgentStatus: imcJoinedAgentStatus, -// Conditions: mcJoinedConditions, -// } -// Eventually(func() error { -// if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { -// return err -// } -// if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { -// return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) -// } -// return nil -// }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) -// }) -//}) diff --git a/test/e2eJoinLeavePlacement/e2e_test.go b/test/e2eJoinLeavePlacement/e2e_test.go index 03d9a2074..c42b78935 100644 --- a/test/e2eJoinLeavePlacement/e2e_test.go +++ b/test/e2eJoinLeavePlacement/e2e_test.go @@ -1,3 +1,7 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ package e2eJoinLeavePlacement import ( @@ -12,7 +16,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1" "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/test/e2e/framework" @@ -92,7 +95,6 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(v1alpha1.AddToScheme(scheme)) - utilruntime.Must(workv1alpha1.AddToScheme(scheme)) utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) } diff --git a/test/e2eJoinLeavePlacement/join_leave_placement_test.go b/test/e2eJoinLeavePlacement/join_leave_placement_test.go index 80341511d..9661a8012 100644 --- a/test/e2eJoinLeavePlacement/join_leave_placement_test.go +++ b/test/e2eJoinLeavePlacement/join_leave_placement_test.go @@ -15,6 +15,7 @@ import ( . "github.com/onsi/gomega" testutils "go.goms.io/fleet/test/e2e/utils" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -22,7 +23,9 @@ import ( ) var ( - mc *v1alpha1.MemberCluster + mc *v1alpha1.MemberCluster + crp *v1alpha1.ClusterResourcePlacement + mcStatusCmpOptions = []cmp.Option{ cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), @@ -34,9 +37,50 @@ var _ = Describe("workload orchestration testing with join/leave", func() { It("Test join and leave with CRP", func() { ctx = context.Background() - //cprName := "join-leave-test" - //labelKey := "fleet.azure.com/name" - //labelValue := "test" + cprName := "join-leave-test" + labelKey := "fleet.azure.com/name" + labelValue := "test" + + By("create the resources to be propagated") + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "jlp-test-cluster-role", + Labels: map[string]string{labelKey: labelValue}, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + }, + }, + } + testutils.CreateClusterRole(*HubCluster, cr) + + By("create the cluster resource placement in the hub cluster") + crp = &v1alpha1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: cprName, + }, + Spec: v1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []v1alpha1.ClusterResourceSelector{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: cr.Labels, + }, + }, + }, + }, + } + testutils.CreateClusterResourcePlacement(*HubCluster, crp) + + By("verify the resource is not propagated to member cluster") + Consistently(func() bool { + return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) By("deploy member cluster in the hub cluster") identity := rbacv1.Subject{ @@ -70,5 +114,50 @@ var _ = Describe("workload orchestration testing with join/leave", func() { } return nil }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + + By("verify that the cluster resource placement is applied") + testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) + + By("verify the resource is propagated to member cluster") + Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName) + + By("update member cluster in the hub cluster to leave") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateLeave + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("verify that member cluster is marked as left") + wantMCStatus = v1alpha1.MemberClusterStatus{ + AgentStatus: imcLeftAgentStatus, + Conditions: mcLeftConditions, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) + + By("verify that the resource is still on the member cluster") + Consistently(func() error { + return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) + + By("delete the crp from the hub") + testutils.DeleteClusterResourcePlacement(*HubCluster, crp) + + By("verify that the resource is still on the member cluster") + Consistently(func() error { + return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) + + By("delete cluster role on hub cluster") + testutils.DeleteClusterRole(*HubCluster, cr) + + By("delete member cluster") + testutils.DeleteMemberCluster(ctx, *HubCluster, mc) }) }) From 0e2e5bb6f060552adbac66fcc08cc53d9713d0ff Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 27 Sep 2022 09:28:22 +0530 Subject: [PATCH 4/5] Util for checking MC,IMC status --- test/e2e/e2e_test.go | 40 ++----------------- test/e2e/utils/helper.go | 27 +++++++++++++ .../join_leave_placement_test.go | 26 ++---------- 3 files changed, 35 insertions(+), 58 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 7afa754e7..0ac45450a 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -196,15 +196,7 @@ var _ = BeforeSuite(func() { By("check if internal member cluster status is updated to Joined") wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcJoinedAgentStatus} - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) - } - return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) + testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, wantIMCStatus, imc, imcStatusCmpOptions) By("check if member cluster status is updated to Joined") wantMCStatus := v1alpha1.MemberClusterStatus{ @@ -212,15 +204,7 @@ var _ = BeforeSuite(func() { Conditions: mcJoinedConditions, ResourceUsage: imc.Status.ResourceUsage, } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) }) var _ = AfterSuite(func() { @@ -231,15 +215,7 @@ var _ = AfterSuite(func() { By("check if internal member cluster status is updated to Left") wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcLeftAgentStatus} - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) - } - return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) + testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, wantIMCStatus, imc, imcStatusCmpOptions) By("check if member cluster status is updated to Left") wantMCStatus := v1alpha1.MemberClusterStatus{ @@ -247,15 +223,7 @@ var _ = AfterSuite(func() { Conditions: mcLeftConditions, ResourceUsage: imc.Status.ResourceUsage, } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) testutils.DeleteNamespace(*MemberCluster, memberNamespace) testutils.DeleteNamespace(*HubCluster, workNamespace) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index 9bed8a2c3..fcd3105e0 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -7,6 +7,7 @@ package utils import ( "context" "fmt" + "github.com/google/go-cmp/cmp" "time" // Lint check prohibits non "_test" ending files to have dot imports for ginkgo / gomega. @@ -43,6 +44,32 @@ func DeleteMemberCluster(ctx context.Context, cluster framework.Cluster, mc *v1a }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for member cluster %s to be deleted in %s cluster", mc.Name, cluster.ClusterName) } +// CheckMemberClusterStatus is used to check member cluster status. +func CheckMemberClusterStatus(ctx context.Context, cluster framework.Cluster, wantMCStatus v1alpha1.MemberClusterStatus, mc *v1alpha1.MemberCluster, mcStatusCmpOptions []cmp.Option) { + gomega.Eventually(func() error { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait member cluster %s to have status %s", mc.Name, wantMCStatus) +} + +// CheckInternalMemberClusterStatus is used to check member cluster status. +func CheckInternalMemberClusterStatus(ctx context.Context, cluster framework.Cluster, wantIMCStatus v1alpha1.InternalMemberClusterStatus, imc *v1alpha1.InternalMemberCluster, imcStatusCmpOptions []cmp.Option) { + gomega.Eventually(func() error { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) + } + return nil + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) +} + // CreateClusterRole create cluster role in the hub cluster. func CreateClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { ginkgo.By(fmt.Sprintf("Creating ClusterRole (%s)", cr.Name), func() { diff --git a/test/e2eJoinLeavePlacement/join_leave_placement_test.go b/test/e2eJoinLeavePlacement/join_leave_placement_test.go index 9661a8012..84ec812ac 100644 --- a/test/e2eJoinLeavePlacement/join_leave_placement_test.go +++ b/test/e2eJoinLeavePlacement/join_leave_placement_test.go @@ -7,8 +7,6 @@ package e2eJoinLeavePlacement import ( "context" - "fmt" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" @@ -37,7 +35,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { It("Test join and leave with CRP", func() { ctx = context.Background() - cprName := "join-leave-test" + crpName := "join-leave-test" labelKey := "fleet.azure.com/name" labelValue := "test" @@ -60,7 +58,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { By("create the cluster resource placement in the hub cluster") crp = &v1alpha1.ClusterResourcePlacement{ ObjectMeta: metav1.ObjectMeta{ - Name: cprName, + Name: crpName, }, Spec: v1alpha1.ClusterResourcePlacementSpec{ ResourceSelectors: []v1alpha1.ClusterResourceSelector{ @@ -105,15 +103,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { AgentStatus: imcJoinedAgentStatus, Conditions: mcJoinedConditions, } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) By("verify that the cluster resource placement is applied") testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) @@ -131,15 +121,7 @@ var _ = Describe("workload orchestration testing with join/leave", func() { AgentStatus: imcLeftAgentStatus, Conditions: mcLeftConditions, } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) By("verify that the resource is still on the member cluster") Consistently(func() error { From 25bab973f85fbd0f1c415e0f1b98ed1b91f38dee Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 27 Sep 2022 09:34:06 +0530 Subject: [PATCH 5/5] Fix imports --- test/e2e/utils/helper.go | 4 ++-- test/e2eJoinLeavePlacement/join_leave_placement_test.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index fcd3105e0..9e88fd545 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -7,10 +7,10 @@ package utils import ( "context" "fmt" - "github.com/google/go-cmp/cmp" "time" // Lint check prohibits non "_test" ending files to have dot imports for ginkgo / gomega. + "github.com/google/go-cmp/cmp" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -57,7 +57,7 @@ func CheckMemberClusterStatus(ctx context.Context, cluster framework.Cluster, wa }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait member cluster %s to have status %s", mc.Name, wantMCStatus) } -// CheckInternalMemberClusterStatus is used to check member cluster status. +// CheckInternalMemberClusterStatus is used to check internal member cluster status. func CheckInternalMemberClusterStatus(ctx context.Context, cluster framework.Cluster, wantIMCStatus v1alpha1.InternalMemberClusterStatus, imc *v1alpha1.InternalMemberCluster, imcStatusCmpOptions []cmp.Option) { gomega.Eventually(func() error { if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { diff --git a/test/e2eJoinLeavePlacement/join_leave_placement_test.go b/test/e2eJoinLeavePlacement/join_leave_placement_test.go index 84ec812ac..bd402a8d8 100644 --- a/test/e2eJoinLeavePlacement/join_leave_placement_test.go +++ b/test/e2eJoinLeavePlacement/join_leave_placement_test.go @@ -7,6 +7,7 @@ package e2eJoinLeavePlacement import ( "context" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2"