diff --git a/pkg/blueprint/blueprint_handler.go b/pkg/blueprint/blueprint_handler.go index 20238a832..1b092d8fa 100644 --- a/pkg/blueprint/blueprint_handler.go +++ b/pkg/blueprint/blueprint_handler.go @@ -267,6 +267,7 @@ func (b *BaseBlueprintHandler) WaitForKustomizations() error { names[i] = k.Name } + consecutiveFailures := 0 for { select { case <-timeout: @@ -276,15 +277,23 @@ func (b *BaseBlueprintHandler) WaitForKustomizations() error { case <-ticker.C: kubeconfig := os.Getenv("KUBECONFIG") if err := checkGitRepositoryStatus(kubeconfig); err != nil { - spin.Stop() - fmt.Fprintf(os.Stderr, "\033[31m✗\033[0m %s - \033[31mFailed\033[0m\n", message) - return fmt.Errorf("git repository error: %w", err) + consecutiveFailures++ + if consecutiveFailures >= constants.DEFAULT_KUSTOMIZATION_WAIT_MAX_FAILURES { + spin.Stop() + fmt.Fprintf(os.Stderr, "\033[31m✗\033[0m %s - \033[31mFailed\033[0m\n", message) + return fmt.Errorf("git repository error after %d consecutive failures: %w", consecutiveFailures, err) + } + continue } status, err := checkKustomizationStatus(kubeconfig, names) if err != nil { - spin.Stop() - fmt.Fprintf(os.Stderr, "\033[31m✗\033[0m %s - \033[31mFailed\033[0m\n", message) - return fmt.Errorf("kustomization error: %w", err) + consecutiveFailures++ + if consecutiveFailures >= constants.DEFAULT_KUSTOMIZATION_WAIT_MAX_FAILURES { + spin.Stop() + fmt.Fprintf(os.Stderr, "\033[31m✗\033[0m %s - \033[31mFailed\033[0m\n", message) + return fmt.Errorf("kustomization error after %d consecutive failures: %w", consecutiveFailures, err) + } + continue } allReady := true @@ -300,6 +309,9 @@ func (b *BaseBlueprintHandler) WaitForKustomizations() error { fmt.Fprintf(os.Stderr, "\033[32m✔\033[0m %s - \033[32mDone\033[0m\n", message) return nil } + + // Reset failure counter on successful check + consecutiveFailures = 0 } } } diff --git a/pkg/blueprint/blueprint_handler_test.go b/pkg/blueprint/blueprint_handler_test.go index 65d8cfc61..6420bc178 100644 --- a/pkg/blueprint/blueprint_handler_test.go +++ b/pkg/blueprint/blueprint_handler_test.go @@ -3946,4 +3946,146 @@ func TestBaseBlueprintHandler_WaitForKustomizations(t *testing.T) { t.Errorf("expected kustomization error, got %v", err) } }) + + t.Run("RecoverFromGitRepositoryError", func(t *testing.T) { + // Given a blueprint handler with a kustomization + handler := &BaseBlueprintHandler{ + blueprint: blueprintv1alpha1.Blueprint{ + Kustomizations: []blueprintv1alpha1.Kustomization{ + {Name: "k1", Timeout: &metav1.Duration{Duration: 100 * time.Millisecond}}, + }, + }, + } + handler.kustomizationWaitPollInterval = 10 * time.Millisecond + + // And a Git repository status check that fails twice then succeeds + failCount := 0 + origCheckGit := checkGitRepositoryStatus + origCheckKustom := checkKustomizationStatus + defer func() { + checkGitRepositoryStatus = origCheckGit + checkKustomizationStatus = origCheckKustom + }() + checkGitRepositoryStatus = func(string) error { + if failCount < 2 { + failCount++ + return fmt.Errorf("git repo error") + } + return nil + } + checkKustomizationStatus = func(string, []string) (map[string]bool, error) { + return map[string]bool{"k1": true}, nil + } + + // When waiting for kustomizations to be ready + err := handler.WaitForKustomizations() + + // Then no error should be returned + if err != nil { + t.Errorf("expected no error, got %v", err) + } + }) + + t.Run("RecoverFromKustomizationError", func(t *testing.T) { + // Given a blueprint handler with a kustomization + handler := &BaseBlueprintHandler{ + blueprint: blueprintv1alpha1.Blueprint{ + Kustomizations: []blueprintv1alpha1.Kustomization{ + {Name: "k1", Timeout: &metav1.Duration{Duration: 100 * time.Millisecond}}, + }, + }, + } + handler.kustomizationWaitPollInterval = 10 * time.Millisecond + + // And a kustomization status check that fails twice then succeeds + failCount := 0 + origCheckGit := checkGitRepositoryStatus + origCheckKustom := checkKustomizationStatus + defer func() { + checkGitRepositoryStatus = origCheckGit + checkKustomizationStatus = origCheckKustom + }() + checkGitRepositoryStatus = func(string) error { return nil } + checkKustomizationStatus = func(string, []string) (map[string]bool, error) { + if failCount < 2 { + failCount++ + return nil, fmt.Errorf("kustomization error") + } + return map[string]bool{"k1": true}, nil + } + + // When waiting for kustomizations to be ready + err := handler.WaitForKustomizations() + + // Then no error should be returned + if err != nil { + t.Errorf("expected no error, got %v", err) + } + }) + + t.Run("MaxGitRepositoryFailures", func(t *testing.T) { + // Given a blueprint handler with a kustomization + handler := &BaseBlueprintHandler{ + blueprint: blueprintv1alpha1.Blueprint{ + Kustomizations: []blueprintv1alpha1.Kustomization{ + {Name: "k1", Timeout: &metav1.Duration{Duration: 100 * time.Millisecond}}, + }, + }, + } + handler.kustomizationWaitPollInterval = 10 * time.Millisecond + + // And a Git repository status check that always fails + origCheckGit := checkGitRepositoryStatus + origCheckKustom := checkKustomizationStatus + defer func() { + checkGitRepositoryStatus = origCheckGit + checkKustomizationStatus = origCheckKustom + }() + checkGitRepositoryStatus = func(string) error { return fmt.Errorf("git repo error") } + checkKustomizationStatus = func(string, []string) (map[string]bool, error) { + return map[string]bool{"k1": true}, nil + } + + // When waiting for kustomizations to be ready + err := handler.WaitForKustomizations() + + // Then a Git repository error should be returned with failure count + expectedMsg := fmt.Sprintf("after %d consecutive failures", constants.DEFAULT_KUSTOMIZATION_WAIT_MAX_FAILURES) + if err == nil || !strings.Contains(err.Error(), expectedMsg) { + t.Errorf("expected error with failure count, got %v", err) + } + }) + + t.Run("MaxKustomizationFailures", func(t *testing.T) { + // Given a blueprint handler with a kustomization + handler := &BaseBlueprintHandler{ + blueprint: blueprintv1alpha1.Blueprint{ + Kustomizations: []blueprintv1alpha1.Kustomization{ + {Name: "k1", Timeout: &metav1.Duration{Duration: 100 * time.Millisecond}}, + }, + }, + } + handler.kustomizationWaitPollInterval = 10 * time.Millisecond + + // And a kustomization status check that always fails + origCheckGit := checkGitRepositoryStatus + origCheckKustom := checkKustomizationStatus + defer func() { + checkGitRepositoryStatus = origCheckGit + checkKustomizationStatus = origCheckKustom + }() + checkGitRepositoryStatus = func(string) error { return nil } + checkKustomizationStatus = func(string, []string) (map[string]bool, error) { + return nil, fmt.Errorf("kustomization error") + } + + // When waiting for kustomizations to be ready + err := handler.WaitForKustomizations() + + // Then a kustomization error should be returned with failure count + expectedMsg := fmt.Sprintf("after %d consecutive failures", constants.DEFAULT_KUSTOMIZATION_WAIT_MAX_FAILURES) + if err == nil || !strings.Contains(err.Error(), expectedMsg) { + t.Errorf("expected error with failure count, got %v", err) + } + }) } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 7b6e70751..c54d896d5 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -41,6 +41,8 @@ const ( DEFAULT_KUSTOMIZATION_WAIT_TOTAL_TIMEOUT = 10 * time.Minute // Poll interval for CLI WaitForKustomizations DEFAULT_KUSTOMIZATION_WAIT_POLL_INTERVAL = 5 * time.Second + // Maximum number of consecutive failures before giving up + DEFAULT_KUSTOMIZATION_WAIT_MAX_FAILURES = 5 ) // Default AWS settings