fix(kubernetes): Handle "not found" errors during resource deletion

pkg/blueprint/blueprint_handler.go

@@ -398,17 +398,16 @@ func (b *BaseBlueprintHandler) walkAndCollectTemplates(templateDir, templateRoot
 	return nil
 }
 
-// Down orchestrates the controlled teardown of all kustomizations and their associated resources.
-// It follows a specific sequence to ensure safe deletion:
-// 1. Suspends all kustomizations and their associated helmreleases to prevent reconciliation
-// 2. Applies cleanup kustomizations if defined, which handle resource cleanup tasks
-// 3. Waits for cleanup kustomizations to complete their operations
-// 4. Deletes main kustomizations in reverse dependency order
-// 5. Deletes cleanup kustomizations and their namespace
+// Down orchestrates the teardown of all kustomizations and associated resources, skipping "not found" errors.
+// Sequence:
+// 1. Suspend all kustomizations and associated helmreleases to prevent reconciliation (ignoring not found errors)
+// 2. Apply cleanup kustomizations if defined for resource cleanup tasks
+// 3. Wait for cleanup kustomizations to complete
+// 4. Delete main kustomizations in reverse dependency order, skipping not found errors
+// 5. Delete cleanup kustomizations and their namespace, skipping not found errors
 //
-// The function handles dependency resolution through topological sorting to ensure
-// resources are deleted in the correct order. It also manages a dedicated cleanup
-// namespace for cleanup kustomizations when needed.
+// Dependency resolution is handled via topological sorting to ensure correct deletion order.
+// A dedicated cleanup namespace is managed for cleanup kustomizations when required.
 func (b *BaseBlueprintHandler) Down() error {
 	kustomizations := b.getKustomizations()
 	if len(kustomizations) == 0 {
@@ -470,9 +469,11 @@ func (b *BaseBlueprintHandler) Down() error {
 		k := nameToK[name]
 
 		if err := b.kubernetesManager.SuspendKustomization(k.Name, constants.DEFAULT_FLUX_SYSTEM_NAMESPACE); err != nil {
-			spin.Stop()
-			fmt.Fprintf(os.Stderr, "✗%s - \033[31mFailed\033[0m\n", spin.Suffix)
-			return fmt.Errorf("failed to suspend kustomization %s: %w", k.Name, err)
+			if !isNotFoundError(err) {
+				spin.Stop()
+				fmt.Fprintf(os.Stderr, "✗%s - \033[31mFailed\033[0m\n", spin.Suffix)
+				return fmt.Errorf("failed to suspend kustomization %s: %w", k.Name, err)
+			}
 		}
 
 		helmReleases, err := b.kubernetesManager.GetHelmReleasesForKustomization(k.Name, constants.DEFAULT_FLUX_SYSTEM_NAMESPACE)
@@ -484,9 +485,11 @@ func (b *BaseBlueprintHandler) Down() error {
 
 		for _, hr := range helmReleases {
 			if err := b.kubernetesManager.SuspendHelmRelease(hr.Name, hr.Namespace); err != nil {
-				spin.Stop()
-				fmt.Fprintf(os.Stderr, "✗%s - \033[31mFailed\033[0m\n", spin.Suffix)
-				return fmt.Errorf("failed to suspend helmrelease %s in namespace %s: %w", hr.Name, hr.Namespace, err)
+				if !isNotFoundError(err) {
+					spin.Stop()
+					fmt.Fprintf(os.Stderr, "✗%s - \033[31mFailed\033[0m\n", spin.Suffix)
+					return fmt.Errorf("failed to suspend helmrelease %s in namespace %s: %w", hr.Name, hr.Namespace, err)
+				}
 			}
 		}
 	}
@@ -1342,3 +1345,18 @@ func (b *BaseBlueprintHandler) isOCISource(sourceNameOrURL string) bool {
 
 	return false
 }
+
+// isNotFoundError checks if an error is a Kubernetes resource not found error
+// This is used during cleanup to ignore errors when resources don't exist
+func isNotFoundError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errMsg := strings.ToLower(err.Error())
+	// Check for resource not found errors, but not namespace not found errors
+	return (strings.Contains(errMsg, "resource not found") ||
+		strings.Contains(errMsg, "could not find the requested resource") ||
+		strings.Contains(errMsg, "the server could not find the requested resource")) &&
+		!strings.Contains(errMsg, "namespace not found")
+}

pkg/kubernetes/kubernetes_manager.go

@@ -121,7 +121,11 @@ func (k *BaseKubernetesManager) DeleteKustomization(name, namespace string) erro
 		Resource: "kustomizations",
 	}
 
-	return k.client.DeleteResource(gvr, namespace, name, metav1.DeleteOptions{})
+	err := k.client.DeleteResource(gvr, namespace, name, metav1.DeleteOptions{})
+	if err != nil && isNotFoundError(err) {
+		return nil
+	}
+	return err
 }
 
 // WaitForKustomizations waits for kustomizations to be ready
@@ -284,6 +288,9 @@ func (k *BaseKubernetesManager) GetHelmReleasesForKustomization(name, namespace
 
 	obj, err := k.client.GetResource(gvr, namespace, name)
 	if err != nil {
+		if isNotFoundError(err) {
+			return []helmv2.HelmRelease{}, nil
+		}
 		return nil, fmt.Errorf("failed to get kustomization: %w", err)
 	}
 
@@ -311,7 +318,7 @@ func (k *BaseKubernetesManager) GetHelmReleasesForKustomization(name, namespace
 	return helmReleases, nil
 }
 
-// SuspendKustomization suspends a Kustomization using a JSON merge patch
+// SuspendKustomization applies a JSON merge patch to set spec.suspend=true on the specified Kustomization.
 func (k *BaseKubernetesManager) SuspendKustomization(name, namespace string) error {
 	gvr := schema.GroupVersionResource{
 		Group:    "kustomize.toolkit.fluxcd.io",
@@ -323,10 +330,11 @@ func (k *BaseKubernetesManager) SuspendKustomization(name, namespace string) err
 	_, err := k.client.PatchResource(gvr, namespace, name, types.MergePatchType, patch, metav1.PatchOptions{
 		FieldManager: "windsor-cli",
 	})
+
 	return err
 }
 
-// SuspendHelmRelease suspends a Flux HelmRelease using SSA
+// SuspendHelmRelease applies a JSON merge patch to set spec.suspend=true on the specified HelmRelease.
 func (k *BaseKubernetesManager) SuspendHelmRelease(name, namespace string) error {
 	gvr := schema.GroupVersionResource{
 		Group:    "helm.toolkit.fluxcd.io",
@@ -338,6 +346,7 @@ func (k *BaseKubernetesManager) SuspendHelmRelease(name, namespace string) error
 	_, err := k.client.PatchResource(gvr, namespace, name, types.MergePatchType, patch, metav1.PatchOptions{
 		FieldManager: "windsor-cli",
 	})
+
 	return err
 }
 
@@ -643,3 +652,18 @@ func isImmutableConfigMap(obj *unstructured.Unstructured) bool {
 	immutable, ok := spec["immutable"].(bool)
 	return ok && immutable
 }
+
+// isNotFoundError checks if an error is a Kubernetes resource not found error
+// This is used during cleanup to ignore errors when resources don't exist
+func isNotFoundError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errMsg := strings.ToLower(err.Error())
+	// Check for resource not found errors, but not namespace not found errors
+	return (strings.Contains(errMsg, "resource not found") ||
+		strings.Contains(errMsg, "could not find the requested resource") ||
+		strings.Contains(errMsg, "the server could not find the requested resource")) &&
+		!strings.Contains(errMsg, "namespace not found")
+}

pkg/kubernetes/kubernetes_manager_test.go

@@ -220,6 +220,20 @@ func TestBaseKubernetesManager_DeleteKustomization(t *testing.T) {
 			t.Error("Expected error, got nil")
 		}
 	})
+
+	t.Run("KustomizationNotFound", func(t *testing.T) {
+		manager := setup(t)
+		client := NewMockKubernetesClient()
+		client.DeleteResourceFunc = func(gvr schema.GroupVersionResource, namespace, name string, opts metav1.DeleteOptions) error {
+			return fmt.Errorf("the server could not find the requested resource")
+		}
+		manager.client = client
+
+		err := manager.DeleteKustomization("test-kustomization", "test-namespace")
+		if err != nil {
+			t.Errorf("Expected no error for not found resource, got %v", err)
+		}
+	})
 }
 
 func TestBaseKubernetesManager_WaitForKustomizations(t *testing.T) {
@@ -1088,6 +1102,23 @@ func TestBaseKubernetesManager_GetHelmReleasesForKustomization(t *testing.T) {
 		}
 	})
 
+	t.Run("KustomizationNotFound", func(t *testing.T) {
+		manager := setup(t)
+		client := NewMockKubernetesClient()
+		client.GetResourceFunc = func(gvr schema.GroupVersionResource, ns, name string) (*unstructured.Unstructured, error) {
+			return nil, fmt.Errorf("the server could not find the requested resource")
+		}
+		manager.client = client
+
+		releases, err := manager.GetHelmReleasesForKustomization("test-kustomization", "test-namespace")
+		if err != nil {
+			t.Errorf("Expected no error, got %v", err)
+		}
+		if len(releases) != 0 {
+			t.Errorf("Expected 0 releases, got %d", len(releases))
+		}
+	})
+
 	t.Run("FromUnstructuredError", func(t *testing.T) {
 		manager := setup(t)
 		client := NewMockKubernetesClient()
@@ -1153,15 +1184,15 @@ func TestBaseKubernetesManager_SuspendKustomization(t *testing.T) {
 		}
 	})
 
-	t.Run("InvalidName", func(t *testing.T) {
+	t.Run("ResourceNotFound", func(t *testing.T) {
 		manager := setup(t)
 		client := NewMockKubernetesClient()
 		client.PatchResourceFunc = func(gvr schema.GroupVersionResource, namespace, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*unstructured.Unstructured, error) {
 			return nil, fmt.Errorf("resource not found")
 		}
 		manager.client = client
 
-		err := manager.SuspendKustomization("", "test-namespace")
+		err := manager.SuspendKustomization("nonexistent-kustomization", "test-namespace")
 		if err == nil {
 			t.Error("Expected error, got nil")
 		}
@@ -1170,22 +1201,39 @@ func TestBaseKubernetesManager_SuspendKustomization(t *testing.T) {
 		}
 	})
 
-	t.Run("InvalidNamespace", func(t *testing.T) {
+	t.Run("PatchResourceError", func(t *testing.T) {
 		manager := setup(t)
 		client := NewMockKubernetesClient()
 		client.PatchResourceFunc = func(gvr schema.GroupVersionResource, namespace, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*unstructured.Unstructured, error) {
 			return nil, fmt.Errorf("namespace not found")
 		}
 		manager.client = client
 
-		err := manager.SuspendKustomization("test-kustomization", "")
+		err := manager.SuspendKustomization("test-kustomization", "nonexistent-namespace")
 		if err == nil {
 			t.Error("Expected error, got nil")
 		}
 		if !strings.Contains(err.Error(), "namespace not found") {
 			t.Errorf("Expected error containing 'namespace not found', got %v", err)
 		}
 	})
+
+	t.Run("ServerCouldNotFindResource", func(t *testing.T) {
+		manager := setup(t)
+		client := NewMockKubernetesClient()
+		client.PatchResourceFunc = func(gvr schema.GroupVersionResource, namespace, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*unstructured.Unstructured, error) {
+			return nil, fmt.Errorf("the server could not find the requested resource")
+		}
+		manager.client = client
+
+		err := manager.SuspendKustomization("observability", "test-namespace")
+		if err == nil {
+			t.Error("Expected error, got nil")
+		}
+		if !strings.Contains(err.Error(), "the server could not find the requested resource") {
+			t.Errorf("Expected error containing 'the server could not find the requested resource', got %v", err)
+		}
+	})
 }
 
 func TestBaseKubernetesManager_SuspendHelmRelease(t *testing.T) {
@@ -1234,15 +1282,15 @@ func TestBaseKubernetesManager_SuspendHelmRelease(t *testing.T) {
 		}
 	})
 
-	t.Run("InvalidName", func(t *testing.T) {
+	t.Run("ResourceNotFound", func(t *testing.T) {
 		manager := setup(t)
 		client := NewMockKubernetesClient()
 		client.PatchResourceFunc = func(gvr schema.GroupVersionResource, namespace, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*unstructured.Unstructured, error) {
 			return nil, fmt.Errorf("resource not found")
 		}
 		manager.client = client
 
-		err := manager.SuspendHelmRelease("", "test-namespace")
+		err := manager.SuspendHelmRelease("nonexistent-release", "test-namespace")
 		if err == nil {
 			t.Error("Expected error, got nil")
 		}
@@ -1251,22 +1299,39 @@ func TestBaseKubernetesManager_SuspendHelmRelease(t *testing.T) {
 		}
 	})
 
-	t.Run("InvalidNamespace", func(t *testing.T) {
+	t.Run("PatchResourceError", func(t *testing.T) {
 		manager := setup(t)
 		client := NewMockKubernetesClient()
 		client.PatchResourceFunc = func(gvr schema.GroupVersionResource, namespace, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*unstructured.Unstructured, error) {
 			return nil, fmt.Errorf("namespace not found")
 		}
 		manager.client = client
 
-		err := manager.SuspendHelmRelease("test-release", "")
+		err := manager.SuspendHelmRelease("test-release", "nonexistent-namespace")
 		if err == nil {
 			t.Error("Expected error, got nil")
 		}
 		if !strings.Contains(err.Error(), "namespace not found") {
 			t.Errorf("Expected error containing 'namespace not found', got %v", err)
 		}
 	})
+
+	t.Run("ServerCouldNotFindResource", func(t *testing.T) {
+		manager := setup(t)
+		client := NewMockKubernetesClient()
+		client.PatchResourceFunc = func(gvr schema.GroupVersionResource, namespace, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*unstructured.Unstructured, error) {
+			return nil, fmt.Errorf("the server could not find the requested resource")
+		}
+		manager.client = client
+
+		err := manager.SuspendHelmRelease("observability", "test-namespace")
+		if err == nil {
+			t.Error("Expected error, got nil")
+		}
+		if !strings.Contains(err.Error(), "the server could not find the requested resource") {
+			t.Errorf("Expected error containing 'the server could not find the requested resource', got %v", err)
+		}
+	})
 }
 
 func TestBaseKubernetesManager_ApplyGitRepository(t *testing.T) {