diff --git a/Makefile b/Makefile index 69e5a36f9..e2f977ace 100644 --- a/Makefile +++ b/Makefile @@ -271,7 +271,7 @@ docker-build-refresh-token: docker-buildx-builder --tag $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(REFRESH_TOKEN_IMAGE_VERSION) . ## ----------------------------------- -## Cleanup +## Cleanup ## ----------------------------------- .PHONY: clean-bin diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index bf8ca16fc..7afa754e7 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -5,15 +5,22 @@ Licensed under the MIT license. package e2e import ( + "context" "embed" "fmt" "os" "testing" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1" @@ -31,12 +38,99 @@ var ( MemberCluster = framework.NewCluster(memberClusterName, scheme) hubURL string scheme = runtime.NewScheme() + mc *v1alpha1.MemberCluster + imc *v1alpha1.InternalMemberCluster + ctx context.Context // This namespace will store Member cluster-related CRs, such as v1alpha1.MemberCluster - memberNamespace = testutils.NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName)) + memberNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName), + }, + } // This namespace in HubCluster will store v1alpha1.Work to simulate Work-related features in Hub Cluster. - workNamespace = testutils.NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName)) + workNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName), + }, + } + + sortOption = cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }) + imcStatusCmpOptions = []cmp.Option{ + cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), + cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), + sortOption, + } + + mcStatusCmpOptions = []cmp.Option{ + cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), + cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"), + sortOption, + } + + imcJoinedAgentStatus = []v1alpha1.AgentStatus{ + { + Type: v1alpha1.MemberAgent, + Conditions: []metav1.Condition{ + { + Reason: "InternalMemberClusterHealthy", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentHealthy), + }, + { + Reason: "InternalMemberClusterJoined", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentJoined), + }, + }, + }, + } + imcLeftAgentStatus = []v1alpha1.AgentStatus{ + { + Type: v1alpha1.MemberAgent, + Conditions: []metav1.Condition{ + { + Reason: "InternalMemberClusterHealthy", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.AgentHealthy), + }, + { + Reason: "InternalMemberClusterLeft", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.AgentJoined), + }, + }, + }, + } + + mcJoinedConditions = []metav1.Condition{ + { + Reason: "MemberClusterReadyToJoin", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), + }, + { + Reason: "MemberClusterJoined", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ConditionTypeMemberClusterJoined), + }, + } + + mcLeftConditions = []metav1.Condition{ + { + Reason: "MemberClusterNotReadyToJoin", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.ConditionTypeMemberClusterReadyToJoin), + }, + { + Reason: "MemberClusterLeft", + Status: metav1.ConditionFalse, + Type: string(v1alpha1.ConditionTypeMemberClusterJoined), + }, + } //go:embed manifests TestManifestFiles embed.FS @@ -57,25 +151,114 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { kubeconfig := os.Getenv("KUBECONFIG") Expect(kubeconfig).ShouldNot(BeEmpty(), "Failure to retrieve kubeconfig") - hubURL = os.Getenv("HUB_SERVER_URL") Expect(hubURL).ShouldNot(BeEmpty(), "Failure to retrieve Hub URL") // hub setup HubCluster.HubURL = hubURL framework.GetClusterClient(HubCluster) - //member setup MemberCluster.HubURL = hubURL framework.GetClusterClient(MemberCluster) - testutils.CreateNamespace(*MemberCluster, memberNamespace) - testutils.CreateNamespace(*HubCluster, workNamespace) + + ctx = context.Background() + + By("deploy member cluster in the hub cluster") + identity := rbacv1.Subject{ + Name: "member-agent-sa", + Kind: "ServiceAccount", + Namespace: "fleet-system", + } + mc = &v1alpha1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: MemberCluster.ClusterName, + }, + Spec: v1alpha1.MemberClusterSpec{ + Identity: identity, + State: v1alpha1.ClusterStateJoin, + HeartbeatPeriodSeconds: 60, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, mc)).Should(Succeed(), "Failed to create member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("check if internal member cluster created in the hub cluster") + imc = &v1alpha1.InternalMemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: MemberCluster.ClusterName, + Namespace: memberNamespace.Name, + }, + } + Eventually(func() error { + return HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to be synced in %s cluster", imc.Name, HubCluster.ClusterName) + + By("check if internal member cluster status is updated to Joined") + wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcJoinedAgentStatus} + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) + + By("check if member cluster status is updated to Joined") + wantMCStatus := v1alpha1.MemberClusterStatus{ + AgentStatus: imc.Status.AgentStatus, + Conditions: mcJoinedConditions, + ResourceUsage: imc.Status.ResourceUsage, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) }) var _ = AfterSuite(func() { - testutils.DeleteNamespace(*MemberCluster, memberNamespace) + By("update member cluster in the hub cluster") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateLeave + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("check if internal member cluster status is updated to Left") + wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcLeftAgentStatus} + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) + By("check if member cluster status is updated to Left") + wantMCStatus := v1alpha1.MemberClusterStatus{ + AgentStatus: imc.Status.AgentStatus, + Conditions: mcLeftConditions, + ResourceUsage: imc.Status.ResourceUsage, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) + + testutils.DeleteNamespace(*MemberCluster, memberNamespace) testutils.DeleteNamespace(*HubCluster, workNamespace) + By("delete member cluster") + testutils.DeleteMemberCluster(ctx, *HubCluster, mc) }) diff --git a/test/e2e/join_leave_member_test.go b/test/e2e/join_leave_member_test.go deleted file mode 100644 index 0f4da179a..000000000 --- a/test/e2e/join_leave_member_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright (c) Microsoft Corporation. -Licensed under the MIT license. -*/ -package e2e - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "go.goms.io/fleet/apis/v1alpha1" - testutils "go.goms.io/fleet/test/e2e/utils" -) - -var _ = Describe("Join/leave member cluster testing", func() { - var mc *v1alpha1.MemberCluster - var sa *corev1.ServiceAccount - var imc *v1alpha1.InternalMemberCluster - var ctx context.Context - - BeforeEach(func() { - ctx = context.Background() - - sa = testutils.NewServiceAccount(MemberCluster.ClusterName, memberNamespace.Name) - testutils.CreateServiceAccount(*MemberCluster, sa) - - By("deploy member cluster in the hub cluster") - mc = testutils.NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin) - testutils.CreateMemberCluster(*HubCluster, mc) - - By("check if internal member cluster created in the hub cluster") - imc = testutils.NewInternalMemberCluster(MemberCluster.ClusterName, memberNamespace.Name) - testutils.WaitInternalMemberCluster(*HubCluster, imc) - - By("check if member cluster is marked as readyToJoin") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionTrue, 3*testutils.PollTimeout) - }) - - AfterEach(func() { - testutils.DeleteMemberCluster(ctx, *HubCluster, mc) - testutils.DeleteServiceAccount(*MemberCluster, sa) - - }) - - It("Join & Leave flow is successful ", func() { - By("check if internal member cluster condition is updated to Joined") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, 3*testutils.PollTimeout) - - By("check if member cluster condition is updated to Joined") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionTrue, 3*testutils.PollTimeout) - - By("update member cluster in the hub cluster") - testutils.UpdateMemberClusterState(*HubCluster, mc, v1alpha1.ClusterStateLeave) - - By("check if internal member cluster condition is updated to Left") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionFalse, 3*testutils.PollTimeout) - - By("check if member cluster is marked as notReadyToJoin") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionFalse, 3*testutils.PollTimeout) - - By("check if member cluster condition is updated to Left") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionFalse, 3*testutils.PollTimeout) - }) -}) diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go new file mode 100644 index 000000000..875cefaa0 --- /dev/null +++ b/test/e2e/join_leave_placement_test.go @@ -0,0 +1,188 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ +package e2e + +import ( + "context" + "fmt" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "go.goms.io/fleet/apis/v1alpha1" + testutils "go.goms.io/fleet/test/e2e/utils" +) + +// Serial - Ginkgo will guarantee that these specs will never run in parallel with other specs. +// Ordered - Ginkgo will guarantee that specs in an Ordered container will run sequentially, in the order they are written. +// This test cannot be run in parallel with other specs in the suite as it's leaving, joining, leaving and joining again. +var _ = Describe("workload orchestration testing with join/leave", Serial, Ordered, func() { + var ( + crp *v1alpha1.ClusterResourcePlacement + ctx context.Context + + mcStatusCmpOptions = []cmp.Option{ + cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), + cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), + cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }), + } + ) + + It("Test join and leave with CRP", func() { + ctx = context.Background() + cprName := "join-leave-test" + labelKey := "fleet.azure.com/name" + labelValue := "test" + + By("update member cluster in the hub cluster to leave") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateLeave + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("check if member cluster status is updated to Left") + wantMCStatus := v1alpha1.MemberClusterStatus{ + AgentStatus: imcLeftAgentStatus, + Conditions: mcLeftConditions, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + + By("create the resources to be propagated") + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "jlp-test-cluster-role", + Labels: map[string]string{labelKey: labelValue}, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + }, + }, + } + testutils.CreateClusterRole(*HubCluster, cr) + + By("create the cluster resource placement in the hub cluster") + crp = &v1alpha1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: cprName, + }, + Spec: v1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []v1alpha1.ClusterResourceSelector{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: cr.Labels, + }, + }, + }, + }, + } + testutils.CreateClusterResourcePlacement(*HubCluster, crp) + + By("verify the resource is not propagated to member cluster") + Consistently(func() bool { + return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) + + By("update member cluster in the hub cluster to join") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateJoin + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("check if member cluster condition is updated to Joined") + wantMCStatus = v1alpha1.MemberClusterStatus{ + AgentStatus: imcJoinedAgentStatus, + Conditions: mcJoinedConditions, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + + By("verify that the cluster resource placement is applied") + testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) + + By("verify the resource is propagated to member cluster") + Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName) + + By("update member cluster in the hub cluster to leave") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateLeave + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("verify that member cluster is marked as left") + wantMCStatus = v1alpha1.MemberClusterStatus{ + AgentStatus: imcLeftAgentStatus, + Conditions: mcLeftConditions, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) + + By("verify that the resource is still on the member cluster") + Consistently(func() error { + return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) + + By("delete the crp from the hub") + testutils.DeleteClusterResourcePlacement(*HubCluster, crp) + + By("verify that the resource is still on the member cluster") + Consistently(func() error { + return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) + + By("delete cluster role on hub cluster") + testutils.DeleteClusterRole(*HubCluster, cr) + + By("update member cluster in the hub cluster to join") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + mc.Spec.State = v1alpha1.ClusterStateJoin + Expect(HubCluster.KubeClient.Update(ctx, mc)).Should(Succeed(), "Failed to update member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) + + By("check if member cluster condition is updated to Joined") + wantMCStatus = v1alpha1.MemberClusterStatus{ + AgentStatus: imcJoinedAgentStatus, + Conditions: mcJoinedConditions, + } + Eventually(func() error { + if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + }) +}) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index b0632fc49..9bed8a2c3 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -35,121 +35,14 @@ var ( PollTimeout = 60 * time.Second ) -// NewMemberCluster return a new member cluster. -func NewMemberCluster(name string, heartbeat int32, state v1alpha1.ClusterState) *v1alpha1.MemberCluster { - identity := rbacv1.Subject{ - Name: name, - Kind: "ServiceAccount", - Namespace: "fleet-system", - } - return &v1alpha1.MemberCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: v1alpha1.MemberClusterSpec{ - Identity: identity, - State: state, - HeartbeatPeriodSeconds: heartbeat, - }, - } -} - -// NewInternalMemberCluster returns a new internal member cluster. -func NewInternalMemberCluster(name, namespace string) *v1alpha1.InternalMemberCluster { - return &v1alpha1.InternalMemberCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } -} - -// NewServiceAccount returns a new service account. -func NewServiceAccount(name, namespace string) *corev1.ServiceAccount { - return &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } -} - -// NewNamespace returns a new namespace. -func NewNamespace(name string) *corev1.Namespace { - return &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - } -} - -// CreateMemberCluster creates MemberCluster and waits for MemberCluster to exist in the hub cluster. -func CreateMemberCluster(cluster framework.Cluster, mc *v1alpha1.MemberCluster) { - ginkgo.By(fmt.Sprintf("Creating MemberCluster(%s)", mc.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), mc) - gomega.Expect(err).Should(gomega.Succeed()) - }) - klog.Infof("Waiting for MemberCluster(%s) to be synced", mc.Name) - gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: mc.Name, Namespace: ""}, mc) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) -} - -// UpdateMemberClusterState updates MemberCluster in the hub cluster. -func UpdateMemberClusterState(cluster framework.Cluster, mc *v1alpha1.MemberCluster, state v1alpha1.ClusterState) { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: mc.Name, Namespace: ""}, mc) - gomega.Expect(err).Should(gomega.Succeed()) - mc.Spec.State = state - err = cluster.KubeClient.Update(context.TODO(), mc) - gomega.Expect(err).Should(gomega.Succeed()) -} - // DeleteMemberCluster deletes MemberCluster in the hub cluster. func DeleteMemberCluster(ctx context.Context, cluster framework.Cluster, mc *v1alpha1.MemberCluster) { - ginkgo.By(fmt.Sprintf("Deleting MemberCluster(%s)", mc.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), mc) - gomega.Expect(err).Should(gomega.Succeed()) - }) - + gomega.Expect(cluster.KubeClient.Delete(ctx, mc)).Should(gomega.Succeed(), "Failed to delete member cluster %s in %s cluster", mc.Name, cluster.ClusterName) gomega.Eventually(func() bool { return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)) }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for member cluster %s to be deleted in %s cluster", mc.Name, cluster.ClusterName) } -// WaitConditionMemberCluster waits for MemberCluster to present on th hub cluster with a specific condition. -func WaitConditionMemberCluster(cluster framework.Cluster, mc *v1alpha1.MemberCluster, conditionType v1alpha1.MemberClusterConditionType, status metav1.ConditionStatus, customTimeout time.Duration) { - klog.Infof("Waiting for MemberCluster(%s) condition(%s) status(%s) to be synced", mc.Name, conditionType, status) - gomega.Eventually(func() bool { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: mc.Name, Namespace: ""}, mc) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - cond := mc.GetCondition(string(conditionType)) - return cond != nil && cond.Status == status - }, customTimeout, PollInterval).Should(gomega.Equal(true)) -} - -// WaitInternalMemberCluster waits for InternalMemberCluster to present on th hub cluster. -func WaitInternalMemberCluster(cluster framework.Cluster, imc *v1alpha1.InternalMemberCluster) { - klog.Infof("Waiting for InternalMemberCluster(%s) to be synced in the %s cluster", imc.Name, cluster.ClusterName) - gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) -} - -// WaitConditionInternalMemberCluster waits for InternalMemberCluster to present on the hub cluster with a specific condition. -// Allowing custom timeout as for join cond it needs longer than defined PollTimeout for the member agent to finish joining. -func WaitConditionInternalMemberCluster(cluster framework.Cluster, imc *v1alpha1.InternalMemberCluster, conditionType v1alpha1.AgentConditionType, status metav1.ConditionStatus, customTimeout time.Duration) { - klog.Infof("Waiting for InternalMemberCluster(%s) condition(%s) status(%s) to be synced in the %s cluster", imc.Name, conditionType, status, cluster.ClusterName) - gomega.Eventually(func() bool { - if err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { - return false - } - cond := imc.GetConditionWithType(v1alpha1.MemberAgent, string(conditionType)) - return cond != nil && cond.Status == status - }, customTimeout, PollInterval).Should(gomega.Equal(true)) -} - // CreateClusterRole create cluster role in the hub cluster. func CreateClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { ginkgo.By(fmt.Sprintf("Creating ClusterRole (%s)", cr.Name), func() { @@ -244,22 +137,6 @@ func DeleteNamespace(cluster framework.Cluster, ns *corev1.Namespace) { }) } -// CreateServiceAccount create serviceaccount. -func CreateServiceAccount(cluster framework.Cluster, sa *corev1.ServiceAccount) { - ginkgo.By(fmt.Sprintf("Creating ServiceAccount(%s)", sa.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), sa) - gomega.Expect(err).Should(gomega.Succeed()) - }) -} - -// DeleteServiceAccount delete serviceaccount. -func DeleteServiceAccount(cluster framework.Cluster, sa *corev1.ServiceAccount) { - ginkgo.By(fmt.Sprintf("Delete ServiceAccount(%s)", sa.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), sa) - gomega.Expect(err).Should(gomega.SatisfyAny(gomega.Succeed(), &utils.NotFoundMatcher{})) - }) -} - // CreateWork creates Work object based on manifest given. func CreateWork(ctx context.Context, hubCluster framework.Cluster, workName, workNamespace string, manifests []workapi.Manifest) workapi.Work { work := workapi.Work{ diff --git a/test/e2e/work_api_e2e_test.go b/test/e2e/work_api_e2e_test.go index ca68402a9..4ae582f11 100644 --- a/test/e2e/work_api_e2e_test.go +++ b/test/e2e/work_api_e2e_test.go @@ -14,7 +14,6 @@ import ( "k8s.io/apimachinery/pkg/types" workapi "sigs.k8s.io/work-api/pkg/apis/v1alpha1" - "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/pkg/utils" testutils "go.goms.io/fleet/test/e2e/utils" ) @@ -29,10 +28,6 @@ var _ = Describe("Work API Controller test", func() { var ( ctx context.Context - - // These variables are used to join the member cluster. - mc *v1alpha1.MemberCluster - // Includes all works applied to the hub cluster. Used for garbage collection. works []workapi.Work @@ -52,25 +47,19 @@ var _ = Describe("Work API Controller test", func() { // This namespace in MemberCluster will store specified test resources created from the Work-api. resourceNamespaceName := "resource-namespace" + utils.RandStr() - resourceNamespace = testutils.NewNamespace(resourceNamespaceName) + resourceNamespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceNamespaceName, + }, + } testutils.CreateNamespace(*MemberCluster, resourceNamespace) - // Member Cluster must join the fleet for Work API to work. - By("deploy member cluster in the hub cluster") - mc = testutils.NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin) - testutils.CreateMemberCluster(*HubCluster, mc) - - By("check if member cluster condition is updated to Joined") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, metav1.ConditionTrue, testutils.PollTimeout) - //Empties the works since they were garbage collected earlier. works = []workapi.Work{} }) AfterEach(func() { testutils.DeleteWork(ctx, *HubCluster, works) - testutils.DeleteMemberCluster(ctx, *HubCluster, mc) - testutils.DeleteNamespace(*MemberCluster, resourceNamespace) }) diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index f28aee95e..4a31ff06c 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -9,65 +9,26 @@ import ( "context" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "go.goms.io/fleet/apis/v1alpha1" testutils "go.goms.io/fleet/test/e2e/utils" ) var _ = Describe("workload orchestration testing", func() { - var mc *v1alpha1.MemberCluster - var imc *v1alpha1.InternalMemberCluster - var sa *corev1.ServiceAccount var crp *v1alpha1.ClusterResourcePlacement var ctx context.Context BeforeEach(func() { ctx = context.Background() - By("prepare resources in member cluster") - // create testing NS in member cluster - sa = testutils.NewServiceAccount(MemberCluster.ClusterName, memberNamespace.Name) - testutils.CreateServiceAccount(*MemberCluster, sa) }) AfterEach(func() { - By("delete the member cluster") - testutils.DeleteMemberCluster(ctx, *HubCluster, mc) - testutils.DeleteServiceAccount(*MemberCluster, sa) testutils.DeleteClusterResourcePlacement(*HubCluster, crp) }) Context("Test Workload Orchestration", func() { - BeforeEach(func() { - By("deploy member cluster in the hub cluster") - mc = testutils.NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin) - testutils.CreateMemberCluster(*HubCluster, mc) - - By("check if internal member cluster created in the hub cluster") - imc = testutils.NewInternalMemberCluster(MemberCluster.ClusterName, memberNamespace.Name) - testutils.WaitInternalMemberCluster(*HubCluster, imc) - - By("check if internal member cluster condition is updated to Joined") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, testutils.PollTimeout) - By("check if member cluster condition is updated to Joined") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionTrue, testutils.PollTimeout) - }) - - AfterEach(func() { - By("update member cluster in the hub cluster") - testutils.UpdateMemberClusterState(*HubCluster, mc, v1alpha1.ClusterStateLeave) - - By("check if internal member cluster condition is updated to Left") - testutils.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionFalse, testutils.PollTimeout) - - By("check if member cluster is marked as notReadyToJoin") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionFalse, testutils.PollTimeout) - }) - It("Apply CRP and check if work gets propagated", func() { workName := "resource-label-selector" labelKey := "fleet.azure.com/name" @@ -120,86 +81,4 @@ var _ = Describe("workload orchestration testing", func() { testutils.DeleteClusterRole(*HubCluster, cr) }) }) - - It("Test join and leave with CRP", func() { - cprName := "join-leave-test2" - labelKey := "fleet.azure.com/name" - labelValue := "test" - By("create the resources to be propagated") - cr := &rbacv1.ClusterRole{ - ObjectMeta: v1.ObjectMeta{ - Name: "test2", - Labels: map[string]string{labelKey: labelValue}, - }, - Rules: []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list", "watch"}, - APIGroups: []string{""}, - Resources: []string{"secrets"}, - }, - }, - } - testutils.CreateClusterRole(*HubCluster, cr) - - By("create the cluster resource placement in the hub cluster") - crp = &v1alpha1.ClusterResourcePlacement{ - ObjectMeta: v1.ObjectMeta{ - Name: cprName, - }, - Spec: v1alpha1.ClusterResourcePlacementSpec{ - ResourceSelectors: []v1alpha1.ClusterResourceSelector{ - { - Group: "rbac.authorization.k8s.io", - Version: "v1", - Kind: "ClusterRole", - LabelSelector: &v1.LabelSelector{ - MatchLabels: cr.Labels, - }, - }, - }, - }, - } - testutils.CreateClusterResourcePlacement(*HubCluster, crp) - - By("verify the resource is not propagated to member cluster") - Consistently(func() error { - return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - }, testutils.PollTimeout, testutils.PollInterval).ShouldNot(Succeed()) - - By("add member cluster in the hub cluster") - mc = testutils.NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin) - testutils.CreateMemberCluster(*HubCluster, mc) - - By("check if member cluster condition is updated to Joined") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionTrue, testutils.PollTimeout) - - By("verify that the cluster resource placement is applied") - testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), v1.ConditionTrue, testutils.PollTimeout) - - By("verify the resource is propagated to member cluster") - Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr)).Should(Succeed()) - - By("mark the member cluster in the hub cluster as leave") - testutils.UpdateMemberClusterState(*HubCluster, mc, v1alpha1.ClusterStateLeave) - - By("verify that member cluster is marked as leave") - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionFalse, testutils.PollTimeout) - testutils.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoined, v1.ConditionFalse, testutils.PollTimeout) - - By("verify that the resource is still on the member cluster") - Consistently(func() error { - return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed()) - - By("delete the crp from the hub") - testutils.DeleteClusterResourcePlacement(*HubCluster, crp) - - By("verify that the resource is still on the member cluster") - Consistently(func() error { - return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed()) - - By("delete cluster role on hub cluster") - testutils.DeleteClusterRole(*HubCluster, cr) - }) })