fix: start health server before manager init to prevent 503 on upgrades

cmd/main.go

@@ -101,6 +101,31 @@ func main() {
 		tlsOpts = append(tlsOpts, disableHTTP2)
 	}
 
+	// Create HealthManager and start health server early so probes are
+	// answered immediately, before the (potentially slow) manager and
+	// controller initialisation.
+	healthManager := health.NewHealthManager()
+	healthManager.Register(health.ComponentCollectorManager)
+	healthManager.Register(health.ComponentBufferQueue)
+	healthManager.Register(health.ComponentDakrTransport)
+	healthManager.Register(health.ComponentMpaServer)
+	healthManager.Register(health.ComponentPrometheus)
+
+	healthServer := health.NewHealthServer(healthManager, probeAddr)
+	if err := healthServer.Start(); err != nil {
+		setupLog.Error(err, "unable to start health server")
+		os.Exit(1)
+	}
+	defer func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := healthServer.Stop(ctx); err != nil {
+			setupLog.Error(err, "error stopping health server")
+		}
+	}()
+
+	setupLog.Info("health server started, initializing manager")
+
 	webhookServer := webhook.NewServer(webhook.Options{
 		TLSOpts: tlsOpts,
 	})
@@ -148,7 +173,7 @@ func main() {
 	}
 
 	// Setup the env-based controller instead of the standard controller
-	envController, err := controller.NewEnvBasedController(mgr, reconcileInterval, mpaServerPort)
+	envController, err := controller.NewEnvBasedController(mgr, healthManager, reconcileInterval, mpaServerPort)
 	if err != nil {
 		setupLog.Error(err, "unable to create environment-based controller")
 		os.Exit(1)
@@ -164,20 +189,6 @@ func main() {
 	// No need to add the standard controller with kubebuilder:scaffold:builder
 	// The env-based controller doesn't rely on CRDs
 
-	// New health server from health package
-	healthServer := health.NewHealthServer(envController.Reconciler.HealthManager, probeAddr)
-	if err := healthServer.Start(); err != nil {
-		setupLog.Error(err, "unable to start health server")
-		os.Exit(1)
-	}
-	defer func() {
-		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-		defer cancel()
-		if err := healthServer.Stop(ctx); err != nil {
-			setupLog.Error(err, "error stopping health server")
-		}
-	}()
-
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")

internal/controller/custom.go

@@ -62,7 +62,7 @@ type EnvBasedController struct {
 }
 
 // NewEnvBasedController creates a new environment-based controller
-func NewEnvBasedController(mgr ctrl.Manager, reconcileInterval time.Duration, mpaServerPort int) (*EnvBasedController, error) {
+func NewEnvBasedController(mgr ctrl.Manager, healthManager *health.HealthManager, reconcileInterval time.Duration, mpaServerPort int) (*EnvBasedController, error) {
 	// Set up basic components
 	logger := util.NewLogger("env-controller")
 	zapLogger, err := zap.NewProduction()
@@ -97,14 +97,6 @@ func NewEnvBasedController(mgr ctrl.Manager, reconcileInterval time.Duration, mp
 		return nil, fmt.Errorf("failed to create apiextensions client: %w", err)
 	}
 
-	// Initialize HealthManager and register components
-	healthManager := health.NewHealthManager()
-	healthManager.Register(health.ComponentCollectorManager)
-	healthManager.Register(health.ComponentBufferQueue)
-	healthManager.Register(health.ComponentDakrTransport)
-	healthManager.Register(health.ComponentMpaServer)
-	healthManager.Register(health.ComponentPrometheus)
-
 	// Create a shared Telemetry metrics instance
 	sharedTelemetryMetrics := collector.NewTelemetryMetrics()