From 65c7aac57fba9f70cc7840e1635e3598353b9dd4 Mon Sep 17 00:00:00 2001 From: Zhiying Lin Date: Wed, 5 Jul 2023 11:04:31 +0800 Subject: [PATCH] test: add integration tests for crp watcher --- .../suite_test.go | 102 ++++++++++++ .../watcher_integration_test.go | 155 ++++++++++++++++++ .../controller_integration_test.go | 58 +++++-- 3 files changed, 301 insertions(+), 14 deletions(-) create mode 100644 pkg/controllers/clusterresourceplacementwatcher/suite_test.go create mode 100644 pkg/controllers/clusterresourceplacementwatcher/watcher_integration_test.go diff --git a/pkg/controllers/clusterresourceplacementwatcher/suite_test.go b/pkg/controllers/clusterresourceplacementwatcher/suite_test.go new file mode 100644 index 000000000..1f92ed975 --- /dev/null +++ b/pkg/controllers/clusterresourceplacementwatcher/suite_test.go @@ -0,0 +1,102 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ +package clusterresourceplacementwatcher + +import ( + "context" + "flag" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "k8s.io/klog/v2/klogr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/test/utils/controller" +) + +var ( + cfg *rest.Config + mgr manager.Manager + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + fakePlacementController *controller.FakeController +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "ClusterResourcePlacement Watcher Suite") +} + +var _ = BeforeSuite(func() { + klog.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("../../../", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).Should(Succeed()) + Expect(cfg).NotTo(BeNil()) + + err = fleetv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + By("construct the k8s client") + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).Should(Succeed()) + Expect(k8sClient).NotTo(BeNil()) + + By("starting the controller manager") + klog.InitFlags(flag.CommandLine) + flag.Parse() + + mgr, err = ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + MetricsBindAddress: "0", + Logger: klogr.NewWithOptions(klogr.WithFormat(klogr.FormatKlog)), + }) + Expect(err).Should(Succeed()) + + fakePlacementController = &controller.FakeController{} + + err = (&Reconciler{ + PlacementController: fakePlacementController, + }).SetupWithManager(mgr) + Expect(err).Should(Succeed()) + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).Should(Succeed(), "failed to run manager") + }() +}) + +var _ = AfterSuite(func() { + defer klog.Flush() + + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).Should(Succeed()) +}) diff --git a/pkg/controllers/clusterresourceplacementwatcher/watcher_integration_test.go b/pkg/controllers/clusterresourceplacementwatcher/watcher_integration_test.go new file mode 100644 index 000000000..3dd2ab8a5 --- /dev/null +++ b/pkg/controllers/clusterresourceplacementwatcher/watcher_integration_test.go @@ -0,0 +1,155 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ +package clusterresourceplacementwatcher + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +const ( + testCRPName = "my-crp" +) + +func clusterResourcePlacementForTest() *fleetv1beta1.ClusterResourcePlacement { + return &fleetv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: testCRPName, + }, + Spec: fleetv1beta1.ClusterResourcePlacementSpec{ + ResourceSelectors: []fleetv1beta1.ClusterResourceSelector{ + { + Group: corev1.GroupName, + Version: "v1", + Kind: "Service", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"region": "east"}, + }, + }, + }, + Policy: &fleetv1beta1.PlacementPolicy{}, + }, + } +} + +var _ = Describe("Test ClusterResourcePlacement Watcher", func() { + const ( + eventuallyTimeout = time.Second * 10 + consistentlyDuration = time.Second * 10 + interval = time.Millisecond * 250 + ) + + var ( + createdCRP = &fleetv1beta1.ClusterResourcePlacement{} + ) + + BeforeEach(func() { + fakePlacementController.ResetQueue() + By("By creating a new clusterResourcePlacement") + crp := clusterResourcePlacementForTest() + Expect(k8sClient.Create(ctx, crp)).Should(Succeed()) + }) + + Context("When creating new clusterResourcePlacement", func() { + AfterEach(func() { + By("By deleting crp") + createdCRP = clusterResourcePlacementForTest() + Expect(k8sClient.Delete(ctx, createdCRP)).Should(Succeed()) + + By("By checking crp") + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, createdCRP)) + }, eventuallyTimeout, interval).Should(BeTrue(), "crp should be deleted") + }) + + It("Should enqueue the event", func() { + By("By checking placement controller queue") + Eventually(func() bool { + return fakePlacementController.Key() == testCRPName + }, eventuallyTimeout, interval).Should(BeTrue(), "placementController should receive the CRP name") + }) + }) + + Context("When updating clusterResourcePlacement", func() { + BeforeEach(func() { + By("By resetting the placement queue") + fakePlacementController.ResetQueue() + }) + AfterEach(func() { + By("By deleting crp") + createdCRP = clusterResourcePlacementForTest() + Expect(k8sClient.Delete(ctx, createdCRP)).Should(Succeed()) + + By("By checking crp") + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, createdCRP)) + }, eventuallyTimeout, interval).Should(BeTrue(), "crp should be deleted") + }) + + It("Updating the spec and it should enqueue the event", func() { + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, createdCRP)).Should(Succeed()) + + By("By updating the clusterResourcePlacement spec") + revisionLimit := int32(3) + createdCRP.Spec.RevisionHistoryLimit = &revisionLimit + Expect(k8sClient.Update(ctx, createdCRP)).Should(Succeed()) + + By("By checking placement controller queue") + Eventually(func() bool { + return fakePlacementController.Key() == testCRPName + }, eventuallyTimeout, interval).Should(BeTrue(), "placementController should receive the CRP name") + }) + + It("Updating the status and it should ignore the event", func() { + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, createdCRP)).Should(Succeed()) + + By("By updating the clusterResourcePlacement status") + newCondition := metav1.Condition{ + Type: string(fleetv1beta1.ClusterResourcePlacementAppliedConditionType), + Status: metav1.ConditionTrue, + Reason: "applied", + ObservedGeneration: createdCRP.GetGeneration(), + } + createdCRP.SetConditions(newCondition) + Expect(k8sClient.Status().Update(ctx, createdCRP)).Should(Succeed()) + + By("By checking placement controller queue") + Consistently(func() bool { + return fakePlacementController.Key() == "" + }, consistentlyDuration, interval).Should(BeTrue(), "watcher should ignore the update status event and not enqueue the request into the placementController queue") + }) + }) + + Context("When deleting clusterResourcePlacement", func() { + BeforeEach(func() { + By("By resetting the placement queue") + fakePlacementController.ResetQueue() + }) + + It("Should enqueue the event", func() { + By("By deleting crp") + createdCRP = clusterResourcePlacementForTest() + Expect(k8sClient.Delete(ctx, createdCRP)).Should(Succeed()) + + By("By checking crp") + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, createdCRP)) + }, eventuallyTimeout, interval).Should(BeTrue(), "crp should be deleted") + + By("By checking placement controller queue") + Eventually(func() bool { + return fakePlacementController.Key() == testCRPName + }, eventuallyTimeout, interval).Should(BeTrue(), "placementController should receive the CRP name") + }) + }) +}) diff --git a/pkg/controllers/clustershedulingpolicysnapshot/controller_integration_test.go b/pkg/controllers/clustershedulingpolicysnapshot/controller_integration_test.go index 2120215c9..f89b55997 100644 --- a/pkg/controllers/clustershedulingpolicysnapshot/controller_integration_test.go +++ b/pkg/controllers/clustershedulingpolicysnapshot/controller_integration_test.go @@ -11,6 +11,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -40,9 +41,9 @@ func policySnapshot() *fleetv1beta1.ClusterSchedulingPolicySnapshot { var _ = Describe("Test clusterSchedulingPolicySnapshot Controller", func() { const ( - timeout = time.Second * 10 - duration = time.Second * 10 - interval = time.Millisecond * 250 + eventuallyTimeout = time.Second * 10 + consistentlyDuration = time.Second * 10 + interval = time.Millisecond * 250 ) var ( @@ -59,65 +60,94 @@ var _ = Describe("Test clusterSchedulingPolicySnapshot Controller", func() { Context("When creating new clusterSchedulingPolicySnapshot", func() { AfterEach(func() { By("By deleting snapshot") - createdSnapshot := policySnapshot() + createdSnapshot = policySnapshot() Expect(k8sClient.Delete(ctx, createdSnapshot)).Should(Succeed()) By("By checking snapshot") Eventually(func() bool { return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testSnapshotName}, createdSnapshot)) - }, duration, interval).Should(BeTrue(), "snapshot should be deleted") + }, eventuallyTimeout, interval).Should(BeTrue(), "snapshot should be deleted") }) It("Should ignore the event", func() { By("By checking placement controller queue") Consistently(func() bool { return fakePlacementController.Key() == "" - }, duration, interval).Should(BeTrue(), "controller should ignore the create event and not enqueue the request into the placementController queue") + }, consistentlyDuration, interval).Should(BeTrue(), "controller should ignore the create event and not enqueue the request into the placementController queue") }) }) Context("When updating clusterSchedulingPolicySnapshot", func() { + BeforeEach(func() { + By("By resetting the placement queue") + fakePlacementController.ResetQueue() + }) + AfterEach(func() { By("By deleting snapshot") - createdSnapshot := policySnapshot() + createdSnapshot = policySnapshot() Expect(k8sClient.Delete(ctx, createdSnapshot)).Should(Succeed()) By("By checking snapshot") Eventually(func() bool { return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testSnapshotName}, createdSnapshot)) - }, duration, interval).Should(BeTrue(), "snapshot should be deleted") + }, eventuallyTimeout, interval).Should(BeTrue(), "snapshot should be deleted") }) - It("Should enqueue the event", func() { + It("Updating the spec and should enqueue the event", func() { Expect(k8sClient.Get(ctx, types.NamespacedName{Name: testSnapshotName}, createdSnapshot)).Should(Succeed()) - By("By updating the clusterSchedulingPolicySnapshot") + By("By updating the clusterSchedulingPolicySnapshot spec") createdSnapshot.Spec.PolicyHash = []byte("modified-hash") Expect(k8sClient.Update(ctx, createdSnapshot)).Should(Succeed()) By("By checking placement controller queue") Eventually(func() bool { return fakePlacementController.Key() == testCRPName - }, timeout, interval).Should(BeTrue(), "placementController should receive the CRP name") + }, eventuallyTimeout, interval).Should(BeTrue(), "placementController should receive the CRP name") + }) + + It("Updating the status and should enqueue the event", func() { + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: testSnapshotName}, createdSnapshot)).Should(Succeed()) + + By("By updating the clusterSchedulingPolicySnapshot status") + newCondition := metav1.Condition{ + Type: string(fleetv1beta1.PolicySnapshotScheduled), + Status: metav1.ConditionTrue, + Reason: "scheduled", + ObservedGeneration: createdSnapshot.GetGeneration(), + } + meta.SetStatusCondition(&createdSnapshot.Status.Conditions, newCondition) + Expect(k8sClient.Status().Update(ctx, createdSnapshot)).Should(Succeed()) + + By("By checking placement controller queue") + Eventually(func() bool { + return fakePlacementController.Key() == testCRPName + }, eventuallyTimeout, interval).Should(BeTrue(), "placementController should receive the CRP name") }) }) Context("When deleting clusterSchedulingPolicySnapshot", func() { + BeforeEach(func() { + By("By resetting the placement queue") + fakePlacementController.ResetQueue() + }) + It("Should ignore the event", func() { By("By deleting snapshot") - createdSnapshot := policySnapshot() + createdSnapshot = policySnapshot() Expect(k8sClient.Delete(ctx, createdSnapshot)).Should(Succeed()) By("By checking snapshot") Eventually(func() bool { return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testSnapshotName}, createdSnapshot)) - }, duration, interval).Should(BeTrue(), "snapshot should be deleted") + }, eventuallyTimeout, interval).Should(BeTrue(), "snapshot should be deleted") By("By checking placement controller queue") Consistently(func() bool { return fakePlacementController.Key() == "" - }, duration, interval).Should(BeTrue(), "controller should ignore the delete event and not enqueue the request into the placementController queue") + }, consistentlyDuration, interval).Should(BeTrue(), "controller should ignore the delete event and not enqueue the request into the placementController queue") }) }) })