From 738c39a67beb7ee6510c8ba4e76867ee87dd7b55 Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Mon, 26 May 2025 00:32:17 -0400 Subject: [PATCH 1/2] feat: enhance blueprint Down with resource suspension --- pkg/blueprint/blueprint_handler.go | 170 ++++++++++++++-- pkg/blueprint/blueprint_handler_test.go | 257 ++++++++++++++++++++++++ 2 files changed, 410 insertions(+), 17 deletions(-) diff --git a/pkg/blueprint/blueprint_handler.go b/pkg/blueprint/blueprint_handler.go index 6a73c9493..162fabb3d 100644 --- a/pkg/blueprint/blueprint_handler.go +++ b/pkg/blueprint/blueprint_handler.go @@ -557,20 +557,28 @@ func (b *BaseBlueprintHandler) SetKustomizations(kustomizations []blueprintv1alp return nil } -// Down tears down all kustomizations in the correct order, running cleanup kustomizations if defined. +// Down orchestrates the controlled teardown of all kustomizations and their associated resources. +// It follows a specific sequence to ensure safe deletion: +// 1. Suspends all kustomizations and their associated helmreleases to prevent reconciliation +// 2. Applies cleanup kustomizations if defined, which handle resource cleanup tasks +// 3. Waits for cleanup kustomizations to complete their operations +// 4. Deletes main kustomizations in reverse dependency order +// 5. Deletes cleanup kustomizations and their namespace +// +// The function handles dependency resolution through topological sorting to ensure +// resources are deleted in the correct order. It also manages a dedicated cleanup +// namespace for cleanup kustomizations when needed. func (b *BaseBlueprintHandler) Down() error { kustomizations := b.GetKustomizations() if len(kustomizations) == 0 { return nil } - // Build dependency graph deps := make(map[string][]string) for _, k := range kustomizations { deps[k.Name] = k.DependsOn } - // Topological sort (reverse order) var sorted []string visited := make(map[string]bool) var visit func(string) @@ -587,7 +595,6 @@ func (b *BaseBlueprintHandler) Down() error { for _, k := range kustomizations { visit(k.Name) } - // Reverse for teardown order for i, j := 0, len(sorted)-1; i < j; i, j = i+1, j-1 { sorted[i], sorted[j] = sorted[j], sorted[i] } @@ -597,7 +604,6 @@ func (b *BaseBlueprintHandler) Down() error { nameToK[k.Name] = k } - // Check if we need cleanup namespace needsCleanupNamespace := false for _, k := range kustomizations { if len(k.Cleanup) > 0 { @@ -606,13 +612,31 @@ func (b *BaseBlueprintHandler) Down() error { } } - // Create cleanup namespace if needed if needsCleanupNamespace { if err := b.createManagedNamespace("system-cleanup"); err != nil { return fmt.Errorf("failed to create system-cleanup namespace: %w", err) } } + for _, name := range sorted { + k := nameToK[name] + + if err := b.suspendKustomization(k.Name, constants.DEFAULT_FLUX_SYSTEM_NAMESPACE); err != nil { + return fmt.Errorf("failed to suspend kustomization %s: %w", k.Name, err) + } + + helmReleases, err := b.getHelmReleasesForKustomization(k.Name, constants.DEFAULT_FLUX_SYSTEM_NAMESPACE) + if err != nil { + return fmt.Errorf("failed to get helmreleases for kustomization %s: %w", k.Name, err) + } + + for _, hr := range helmReleases { + if err := b.suspendHelmRelease(hr.Name, hr.Namespace); err != nil { + return fmt.Errorf("failed to suspend helmrelease %s in namespace %s: %w", hr.Name, hr.Namespace, err) + } + } + } + var cleanupNames []string for _, name := range sorted { k := nameToK[name] @@ -635,25 +659,28 @@ func (b *BaseBlueprintHandler) Down() error { } cleanupNames = append(cleanupNames, cleanupKustomization.Name) } - // Delete the main kustomization - if err := b.deleteKustomization(k.Name, constants.DEFAULT_FLUX_SYSTEM_NAMESPACE); err != nil { - return fmt.Errorf("failed to delete kustomization %s: %w", k.Name, err) - } } if len(cleanupNames) > 0 { if err := b.WaitForKustomizations("📐 Deploying cleanup kustomizations", cleanupNames...); err != nil { return fmt.Errorf("failed waiting for cleanup kustomizations: %w", err) } + } + + for _, name := range sorted { + k := nameToK[name] + if err := b.deleteKustomization(k.Name, constants.DEFAULT_FLUX_SYSTEM_NAMESPACE); err != nil { + return fmt.Errorf("failed to delete kustomization %s: %w", k.Name, err) + } + } - // Delete cleanup kustomizations + if len(cleanupNames) > 0 { for _, cname := range cleanupNames { if err := b.deleteKustomization(cname, constants.DEFAULT_FLUX_SYSTEM_NAMESPACE); err != nil { return fmt.Errorf("failed to delete cleanup kustomization %s: %w", cname, err) } } - // Delete the cleanup namespace if err := b.deleteNamespace("system-cleanup"); err != nil { return fmt.Errorf("failed to delete system-cleanup namespace: %w", err) } @@ -666,7 +693,11 @@ func (b *BaseBlueprintHandler) Down() error { // Private Methods // ============================================================================= -// applyKustomization creates or updates a Kustomization resource in the cluster. +// applyKustomization creates or updates a Kustomization resource in the cluster. The function +// handles source resolution, default value population, and proper API object construction. +// It ensures all required fields are set, including intervals, timeouts, and post-build +// configurations. The function uses a get-then-create-or-update pattern to handle both +// new and existing kustomizations. func (b *BaseBlueprintHandler) applyKustomization(kustomization blueprintv1alpha1.Kustomization, namespace string) error { if kustomization.Source == "" { context := b.configHandler.GetContext() @@ -734,7 +765,8 @@ func (b *BaseBlueprintHandler) applyKustomization(kustomization blueprintv1alpha return kubeClientResourceOperation(kubeconfig, config) } -// deleteKustomization deletes a Kustomization resource from the cluster. +// deleteKustomization deletes a Kustomization resource from the cluster. The function +// sends a DELETE request to remove the kustomization and its associated resources. func (b *BaseBlueprintHandler) deleteKustomization(name string, namespace string) error { kubeconfig := os.Getenv("KUBECONFIG") config := ResourceOperationConfig{ @@ -748,7 +780,8 @@ func (b *BaseBlueprintHandler) deleteKustomization(name string, namespace string return b.deleteResource(kubeconfig, config) } -// deleteResource deletes a resource from the cluster using the REST client. +// deleteResource deletes a resource from the cluster using the REST client. The function +// sends a DELETE request to the specified API path and resource. func (b *BaseBlueprintHandler) deleteResource(kubeconfigPath string, config ResourceOperationConfig) error { return kubeClient(kubeconfigPath, KubeRequestConfig{ Method: "DELETE", @@ -858,6 +891,95 @@ func (b *BaseBlueprintHandler) processBlueprintData(data []byte, blueprint *blue return nil } +// suspendKustomization suspends a Flux Kustomization by setting its suspend field to true. +// This prevents the kustomization from reconciling during teardown. The function sends a PATCH +// request to update the kustomization's spec.suspend field. +func (b *BaseBlueprintHandler) suspendKustomization(name, namespace string) error { + patch := map[string]any{ + "spec": map[string]any{ + "suspend": true, + }, + } + patchBytes, err := b.shims.JsonMarshal(patch) + if err != nil { + return fmt.Errorf("failed to marshal patch: %w", err) + } + return kubeClient(os.Getenv("KUBECONFIG"), KubeRequestConfig{ + Method: "PATCH", + ApiPath: "/apis/kustomize.toolkit.fluxcd.io/v1", + Namespace: namespace, + Resource: "kustomizations", + Name: name, + Body: patchBytes, + Headers: map[string]string{ + "Content-Type": "application/merge-patch+json", + }, + }) +} + +// suspendHelmRelease suspends a Flux HelmRelease by setting its suspend field to true. +// This prevents the helmrelease from reconciling during teardown. The function sends a PATCH +// request to update the helmrelease's spec.suspend field. +func (b *BaseBlueprintHandler) suspendHelmRelease(name, namespace string) error { + patch := map[string]any{ + "spec": map[string]any{ + "suspend": true, + }, + } + patchBytes, err := b.shims.JsonMarshal(patch) + if err != nil { + return fmt.Errorf("failed to marshal patch: %w", err) + } + + return kubeClient(os.Getenv("KUBECONFIG"), KubeRequestConfig{ + Method: "PATCH", + ApiPath: "/apis/helm.toolkit.fluxcd.io/v2", + Namespace: namespace, + Resource: "helmreleases", + Name: name, + Body: patchBytes, + Headers: map[string]string{ + "Content-Type": "application/merge-patch+json", + }, + }) +} + +// getHelmReleasesForKustomization retrieves all HelmReleases associated with a Kustomization +// by parsing its inventory entries. The function extracts entries matching the pattern +// {namespace}_{name}_{group}_{kind} to identify helmreleases. Returns a list of helmrelease +// names and their namespaces. +func (b *BaseBlueprintHandler) getHelmReleasesForKustomization(kustomizationName, namespace string) ([]struct{ Name, Namespace string }, error) { + var kustomization kustomizev1.Kustomization + + err := kubeClient(os.Getenv("KUBECONFIG"), KubeRequestConfig{ + Method: "GET", + ApiPath: "/apis/kustomize.toolkit.fluxcd.io/v1", + Namespace: namespace, + Resource: "kustomizations", + Name: kustomizationName, + Response: &kustomization, + }) + if err != nil { + return nil, fmt.Errorf("failed to get kustomization: %w", err) + } + + var helmReleases []struct{ Name, Namespace string } + if kustomization.Status.Inventory == nil { + return helmReleases, nil + } + for _, entry := range kustomization.Status.Inventory.Entries { + parts := strings.Split(entry.ID, "_") + if len(parts) >= 4 && parts[2] == "helm.toolkit.fluxcd.io" && parts[3] == "HelmRelease" { + helmReleases = append(helmReleases, struct{ Name, Namespace string }{ + Name: parts[1], + Namespace: parts[0], + }) + } + } + + return helmReleases, nil +} + // ============================================================================= // Helper Functions // ============================================================================= @@ -1040,12 +1162,19 @@ func (b *BaseBlueprintHandler) createManagedNamespace(name string) error { }, }, } - return kubeClient(kubeconfig, KubeRequestConfig{ + err := kubeClient(kubeconfig, KubeRequestConfig{ Method: "POST", ApiPath: "/api/v1", Resource: "namespaces", Body: ns, }) + if err != nil { + if strings.Contains(err.Error(), "already exists") { + return nil + } + return err + } + return nil } func (b *BaseBlueprintHandler) deleteNamespace(name string) error { @@ -1268,8 +1397,9 @@ type KubeRequestConfig struct { Namespace string Resource string Name string - Body interface{} + Body any Response runtime.Object + Headers map[string]string } // kubeClient performs a Kubernetes API request using the provided configuration. @@ -1313,6 +1443,12 @@ var kubeClient = func(kubeconfigPath string, config KubeRequestConfig) error { req = req.Body(config.Body) } + if config.Headers != nil { + for key, value := range config.Headers { + req = req.SetHeader(key, value) + } + } + result := req.Do(backgroundCtx) if err := result.Error(); err != nil { return err diff --git a/pkg/blueprint/blueprint_handler_test.go b/pkg/blueprint/blueprint_handler_test.go index a99174138..d1f07eaf6 100644 --- a/pkg/blueprint/blueprint_handler_test.go +++ b/pkg/blueprint/blueprint_handler_test.go @@ -2368,6 +2368,26 @@ func TestBaseBlueprintHandler_WaitForKustomizations(t *testing.T) { } func TestBaseBlueprintHandler_Down(t *testing.T) { + // Patch kubeClient to handle all necessary operations + origKubeClient := kubeClient + kubeClient = func(_ string, config KubeRequestConfig) error { + if config.Method == "GET" && config.Resource == "kustomizations" { + kustomization := &kustomizev1.Kustomization{ + Status: kustomizev1.KustomizationStatus{ + Inventory: &kustomizev1.ResourceInventory{ + Entries: []kustomizev1.ResourceRef{}, + }, + }, + } + if config.Response != nil { + *config.Response.(*kustomizev1.Kustomization) = *kustomization + } + return nil + } + return nil + } + defer func() { kubeClient = origKubeClient }() + setup := func(t *testing.T) (*BaseBlueprintHandler, *Mocks) { t.Helper() mocks := setupMocks(t) @@ -2847,3 +2867,240 @@ func TestBaseBlueprintHandler_Down(t *testing.T) { } }) } + +func TestBaseBlueprintHandler_SuspendKustomization(t *testing.T) { + setup := func(t *testing.T) (*BaseBlueprintHandler, *Mocks) { + t.Helper() + mocks := setupMocks(t) + handler := NewBlueprintHandler(mocks.Injector) + handler.shims = mocks.Shims + err := handler.Initialize() + if err != nil { + t.Fatalf("Failed to initialize handler: %v", err) + } + return handler, mocks + } + + t.Run("Success", func(t *testing.T) { + handler, _ := setup(t) + baseHandler := handler + + // Patch kubeClient to verify correct request + origKubeClient := kubeClient + defer func() { kubeClient = origKubeClient }() + var capturedConfig KubeRequestConfig + kubeClient = func(_ string, config KubeRequestConfig) error { + capturedConfig = config + return nil + } + + // When suspending a kustomization + err := baseHandler.suspendKustomization("test-kustomization", "test-namespace") + + // Then no error should be returned + if err != nil { + t.Errorf("expected nil error, got %v", err) + } + + // And the request should be correct + if capturedConfig.Method != "PATCH" { + t.Errorf("expected Method 'PATCH', got '%s'", capturedConfig.Method) + } + if capturedConfig.ApiPath != "/apis/kustomize.toolkit.fluxcd.io/v1" { + t.Errorf("expected ApiPath '/apis/kustomize.toolkit.fluxcd.io/v1', got '%s'", capturedConfig.ApiPath) + } + if capturedConfig.Namespace != "test-namespace" { + t.Errorf("expected Namespace 'test-namespace', got '%s'", capturedConfig.Namespace) + } + if capturedConfig.Resource != "kustomizations" { + t.Errorf("expected Resource 'kustomizations', got '%s'", capturedConfig.Resource) + } + if capturedConfig.Name != "test-kustomization" { + t.Errorf("expected Name 'test-kustomization', got '%s'", capturedConfig.Name) + } + if string(capturedConfig.Body.([]byte)) != `{"spec":{"suspend":true}}` { + t.Errorf("expected Body '{\"spec\":{\"suspend\":true}}', got '%s'", capturedConfig.Body) + } + }) + + t.Run("Error", func(t *testing.T) { + handler, _ := setup(t) + baseHandler := handler + + // Patch kubeClient to return error + origKubeClient := kubeClient + defer func() { kubeClient = origKubeClient }() + kubeClient = func(_ string, _ KubeRequestConfig) error { + return fmt.Errorf("test error") + } + + // When suspending a kustomization + err := baseHandler.suspendKustomization("test-kustomization", "test-namespace") + + // Then error should be returned + if err == nil { + t.Error("expected error, got nil") + } + }) +} + +func TestBaseBlueprintHandler_SuspendHelmRelease(t *testing.T) { + setup := func(t *testing.T) (*BaseBlueprintHandler, *Mocks) { + t.Helper() + mocks := setupMocks(t) + handler := NewBlueprintHandler(mocks.Injector) + handler.shims = mocks.Shims + err := handler.Initialize() + if err != nil { + t.Fatalf("Failed to initialize handler: %v", err) + } + return handler, mocks + } + + t.Run("Success", func(t *testing.T) { + handler, _ := setup(t) + baseHandler := handler + + // Patch kubeClient to verify correct request + origKubeClient := kubeClient + defer func() { kubeClient = origKubeClient }() + var capturedConfig KubeRequestConfig + kubeClient = func(_ string, config KubeRequestConfig) error { + capturedConfig = config + return nil + } + + // When suspending a helmrelease + err := baseHandler.suspendHelmRelease("test-helmrelease", "test-namespace") + + // Then no error should be returned + if err != nil { + t.Errorf("expected nil error, got %v", err) + } + + // And the request should be correct + if capturedConfig.Method != "PATCH" { + t.Errorf("expected Method 'PATCH', got '%s'", capturedConfig.Method) + } + if capturedConfig.ApiPath != "/apis/helm.toolkit.fluxcd.io/v2beta1" { + t.Errorf("expected ApiPath '/apis/helm.toolkit.fluxcd.io/v2beta1', got '%s'", capturedConfig.ApiPath) + } + if capturedConfig.Namespace != "test-namespace" { + t.Errorf("expected Namespace 'test-namespace', got '%s'", capturedConfig.Namespace) + } + if capturedConfig.Resource != "helmreleases" { + t.Errorf("expected Resource 'helmreleases', got '%s'", capturedConfig.Resource) + } + if capturedConfig.Name != "test-helmrelease" { + t.Errorf("expected Name 'test-helmrelease', got '%s'", capturedConfig.Name) + } + if string(capturedConfig.Body.([]byte)) != `{"spec":{"suspend":true}}` { + t.Errorf("expected Body '{\"spec\":{\"suspend\":true}}', got '%s'", capturedConfig.Body) + } + }) + + t.Run("Error", func(t *testing.T) { + handler, _ := setup(t) + baseHandler := handler + + // Patch kubeClient to return error + origKubeClient := kubeClient + defer func() { kubeClient = origKubeClient }() + kubeClient = func(_ string, _ KubeRequestConfig) error { + return fmt.Errorf("test error") + } + + // When suspending a helmrelease + err := baseHandler.suspendHelmRelease("test-helmrelease", "test-namespace") + + // Then error should be returned + if err == nil { + t.Error("expected error, got nil") + } + }) +} + +func TestBaseBlueprintHandler_GetHelmReleasesForKustomization(t *testing.T) { + setup := func(t *testing.T) (*BaseBlueprintHandler, *Mocks) { + t.Helper() + mocks := setupMocks(t) + handler := NewBlueprintHandler(mocks.Injector) + handler.shims = mocks.Shims + err := handler.Initialize() + if err != nil { + t.Fatalf("Failed to initialize handler: %v", err) + } + return handler, mocks + } + + t.Run("Success", func(t *testing.T) { + handler, _ := setup(t) + baseHandler := handler + + // Patch kubeClient to return kustomization with helmreleases + origKubeClient := kubeClient + defer func() { kubeClient = origKubeClient }() + kubeClient = func(_ string, config KubeRequestConfig) error { + if config.Method == "GET" && config.Resource == "kustomizations" { + kustomization := &kustomizev1.Kustomization{ + Status: kustomizev1.KustomizationStatus{ + Inventory: &kustomizev1.ResourceInventory{ + Entries: []kustomizev1.ResourceRef{ + {ID: "ns1_hr1_helm.toolkit.fluxcd.io_HelmRelease"}, + {ID: "ns2_hr2_helm.toolkit.fluxcd.io_HelmRelease"}, + {ID: "ns3_other_kind_OtherResource"}, + }, + }, + }, + } + response := config.Response.(*kustomizev1.Kustomization) + *response = *kustomization + return nil + } + return nil + } + + // When getting helmreleases for a kustomization + helmReleases, err := baseHandler.getHelmReleasesForKustomization("test-kustomization", "test-namespace") + + // Then no error should be returned + if err != nil { + t.Errorf("expected nil error, got %v", err) + } + + // And the helmreleases should be correct + if len(helmReleases) != 2 { + t.Errorf("expected 2 helmreleases, got %d", len(helmReleases)) + } + + expectedReleases := map[string]string{ + "hr1": "ns1", + "hr2": "ns2", + } + for _, hr := range helmReleases { + if expectedNs, ok := expectedReleases[hr.Name]; !ok || expectedNs != hr.Namespace { + t.Errorf("unexpected helmrelease: %s in namespace %s", hr.Name, hr.Namespace) + } + } + }) + + t.Run("Error", func(t *testing.T) { + handler, _ := setup(t) + baseHandler := handler + + // Patch kubeClient to return error + origKubeClient := kubeClient + defer func() { kubeClient = origKubeClient }() + kubeClient = func(_ string, _ KubeRequestConfig) error { + return fmt.Errorf("test error") + } + + // When getting helmreleases for a kustomization + _, err := baseHandler.getHelmReleasesForKustomization("test-kustomization", "test-namespace") + + // Then error should be returned + if err == nil { + t.Error("expected error, got nil") + } + }) +} From ab8fed59405c49fe0726fc696fe87f393ae4f0ce Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Mon, 26 May 2025 07:44:07 -0400 Subject: [PATCH 2/2] v2beta1 => v2 --- pkg/blueprint/blueprint_handler_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/blueprint/blueprint_handler_test.go b/pkg/blueprint/blueprint_handler_test.go index d1f07eaf6..2888e0ec9 100644 --- a/pkg/blueprint/blueprint_handler_test.go +++ b/pkg/blueprint/blueprint_handler_test.go @@ -2982,8 +2982,8 @@ func TestBaseBlueprintHandler_SuspendHelmRelease(t *testing.T) { if capturedConfig.Method != "PATCH" { t.Errorf("expected Method 'PATCH', got '%s'", capturedConfig.Method) } - if capturedConfig.ApiPath != "/apis/helm.toolkit.fluxcd.io/v2beta1" { - t.Errorf("expected ApiPath '/apis/helm.toolkit.fluxcd.io/v2beta1', got '%s'", capturedConfig.ApiPath) + if capturedConfig.ApiPath != "/apis/helm.toolkit.fluxcd.io/v2" { + t.Errorf("expected ApiPath '/apis/helm.toolkit.fluxcd.io/v2', got '%s'", capturedConfig.ApiPath) } if capturedConfig.Namespace != "test-namespace" { t.Errorf("expected Namespace 'test-namespace', got '%s'", capturedConfig.Namespace)