fix: Intermittent wait failure

pkg/blueprint/blueprint_handler.go

@@ -267,6 +267,7 @@ func (b *BaseBlueprintHandler) WaitForKustomizations() error {
 		names[i] = k.Name
 	}
 
+	consecutiveFailures := 0
 	for {
 		select {
 		case <-timeout:
@@ -276,15 +277,23 @@ func (b *BaseBlueprintHandler) WaitForKustomizations() error {
 		case <-ticker.C:
 			kubeconfig := os.Getenv("KUBECONFIG")
 			if err := checkGitRepositoryStatus(kubeconfig); err != nil {
-				spin.Stop()
-				fmt.Fprintf(os.Stderr, "\033[31m✗\033[0m %s - \033[31mFailed\033[0m\n", message)
-				return fmt.Errorf("git repository error: %w", err)
+				consecutiveFailures++
+				if consecutiveFailures >= constants.DEFAULT_KUSTOMIZATION_WAIT_MAX_FAILURES {
+					spin.Stop()
+					fmt.Fprintf(os.Stderr, "\033[31m✗\033[0m %s - \033[31mFailed\033[0m\n", message)
+					return fmt.Errorf("git repository error after %d consecutive failures: %w", consecutiveFailures, err)
+				}
+				continue
 			}
 			status, err := checkKustomizationStatus(kubeconfig, names)
 			if err != nil {
-				spin.Stop()
-				fmt.Fprintf(os.Stderr, "\033[31m✗\033[0m %s - \033[31mFailed\033[0m\n", message)
-				return fmt.Errorf("kustomization error: %w", err)
+				consecutiveFailures++
+				if consecutiveFailures >= constants.DEFAULT_KUSTOMIZATION_WAIT_MAX_FAILURES {
+					spin.Stop()
+					fmt.Fprintf(os.Stderr, "\033[31m✗\033[0m %s - \033[31mFailed\033[0m\n", message)
+					return fmt.Errorf("kustomization error after %d consecutive failures: %w", consecutiveFailures, err)
+				}
+				continue
 			}
 
 			allReady := true
@@ -300,6 +309,9 @@ func (b *BaseBlueprintHandler) WaitForKustomizations() error {
 				fmt.Fprintf(os.Stderr, "\033[32m✔\033[0m %s - \033[32mDone\033[0m\n", message)
 				return nil
 			}
+
+			// Reset failure counter on successful check
+			consecutiveFailures = 0
 		}
 	}
 }

pkg/blueprint/blueprint_handler_test.go

@@ -3946,4 +3946,146 @@ func TestBaseBlueprintHandler_WaitForKustomizations(t *testing.T) {
 			t.Errorf("expected kustomization error, got %v", err)
 		}
 	})
+
+	t.Run("RecoverFromGitRepositoryError", func(t *testing.T) {
+		// Given a blueprint handler with a kustomization
+		handler := &BaseBlueprintHandler{
+			blueprint: blueprintv1alpha1.Blueprint{
+				Kustomizations: []blueprintv1alpha1.Kustomization{
+					{Name: "k1", Timeout: &metav1.Duration{Duration: 100 * time.Millisecond}},
+				},
+			},
+		}
+		handler.kustomizationWaitPollInterval = 10 * time.Millisecond
+
+		// And a Git repository status check that fails twice then succeeds
+		failCount := 0
+		origCheckGit := checkGitRepositoryStatus
+		origCheckKustom := checkKustomizationStatus
+		defer func() {
+			checkGitRepositoryStatus = origCheckGit
+			checkKustomizationStatus = origCheckKustom
+		}()
+		checkGitRepositoryStatus = func(string) error {
+			if failCount < 2 {
+				failCount++
+				return fmt.Errorf("git repo error")
+			}
+			return nil
+		}
+		checkKustomizationStatus = func(string, []string) (map[string]bool, error) {
+			return map[string]bool{"k1": true}, nil
+		}
+
+		// When waiting for kustomizations to be ready
+		err := handler.WaitForKustomizations()
+
+		// Then no error should be returned
+		if err != nil {
+			t.Errorf("expected no error, got %v", err)
+		}
+	})
+
+	t.Run("RecoverFromKustomizationError", func(t *testing.T) {
+		// Given a blueprint handler with a kustomization
+		handler := &BaseBlueprintHandler{
+			blueprint: blueprintv1alpha1.Blueprint{
+				Kustomizations: []blueprintv1alpha1.Kustomization{
+					{Name: "k1", Timeout: &metav1.Duration{Duration: 100 * time.Millisecond}},
+				},
+			},
+		}
+		handler.kustomizationWaitPollInterval = 10 * time.Millisecond
+
+		// And a kustomization status check that fails twice then succeeds
+		failCount := 0
+		origCheckGit := checkGitRepositoryStatus
+		origCheckKustom := checkKustomizationStatus
+		defer func() {
+			checkGitRepositoryStatus = origCheckGit
+			checkKustomizationStatus = origCheckKustom
+		}()
+		checkGitRepositoryStatus = func(string) error { return nil }
+		checkKustomizationStatus = func(string, []string) (map[string]bool, error) {
+			if failCount < 2 {
+				failCount++
+				return nil, fmt.Errorf("kustomization error")
+			}
+			return map[string]bool{"k1": true}, nil
+		}
+
+		// When waiting for kustomizations to be ready
+		err := handler.WaitForKustomizations()
+
+		// Then no error should be returned
+		if err != nil {
+			t.Errorf("expected no error, got %v", err)
+		}
+	})
+
+	t.Run("MaxGitRepositoryFailures", func(t *testing.T) {
+		// Given a blueprint handler with a kustomization
+		handler := &BaseBlueprintHandler{
+			blueprint: blueprintv1alpha1.Blueprint{
+				Kustomizations: []blueprintv1alpha1.Kustomization{
+					{Name: "k1", Timeout: &metav1.Duration{Duration: 100 * time.Millisecond}},
+				},
+			},
+		}
+		handler.kustomizationWaitPollInterval = 10 * time.Millisecond
+
+		// And a Git repository status check that always fails
+		origCheckGit := checkGitRepositoryStatus
+		origCheckKustom := checkKustomizationStatus
+		defer func() {
+			checkGitRepositoryStatus = origCheckGit
+			checkKustomizationStatus = origCheckKustom
+		}()
+		checkGitRepositoryStatus = func(string) error { return fmt.Errorf("git repo error") }
+		checkKustomizationStatus = func(string, []string) (map[string]bool, error) {
+			return map[string]bool{"k1": true}, nil
+		}
+
+		// When waiting for kustomizations to be ready
+		err := handler.WaitForKustomizations()
+
+		// Then a Git repository error should be returned with failure count
+		expectedMsg := fmt.Sprintf("after %d consecutive failures", constants.DEFAULT_KUSTOMIZATION_WAIT_MAX_FAILURES)
+		if err == nil || !strings.Contains(err.Error(), expectedMsg) {
+			t.Errorf("expected error with failure count, got %v", err)
+		}
+	})
+
+	t.Run("MaxKustomizationFailures", func(t *testing.T) {
+		// Given a blueprint handler with a kustomization
+		handler := &BaseBlueprintHandler{
+			blueprint: blueprintv1alpha1.Blueprint{
+				Kustomizations: []blueprintv1alpha1.Kustomization{
+					{Name: "k1", Timeout: &metav1.Duration{Duration: 100 * time.Millisecond}},
+				},
+			},
+		}
+		handler.kustomizationWaitPollInterval = 10 * time.Millisecond
+
+		// And a kustomization status check that always fails
+		origCheckGit := checkGitRepositoryStatus
+		origCheckKustom := checkKustomizationStatus
+		defer func() {
+			checkGitRepositoryStatus = origCheckGit
+			checkKustomizationStatus = origCheckKustom
+		}()
+		checkGitRepositoryStatus = func(string) error { return nil }
+		checkKustomizationStatus = func(string, []string) (map[string]bool, error) {
+			return nil, fmt.Errorf("kustomization error")
+		}
+
+		// When waiting for kustomizations to be ready
+		err := handler.WaitForKustomizations()
+
+		// Then a kustomization error should be returned with failure count
+		expectedMsg := fmt.Sprintf("after %d consecutive failures", constants.DEFAULT_KUSTOMIZATION_WAIT_MAX_FAILURES)
+		if err == nil || !strings.Contains(err.Error(), expectedMsg) {
+			t.Errorf("expected error with failure count, got %v", err)
+		}
+	})
 }

pkg/constants/constants.go

@@ -41,6 +41,8 @@ const (
 	DEFAULT_KUSTOMIZATION_WAIT_TOTAL_TIMEOUT = 10 * time.Minute
 	// Poll interval for CLI WaitForKustomizations
 	DEFAULT_KUSTOMIZATION_WAIT_POLL_INTERVAL = 5 * time.Second
+	// Maximum number of consecutive failures before giving up
+	DEFAULT_KUSTOMIZATION_WAIT_MAX_FAILURES = 5
 )
 
 // Default AWS settings