fix: wait for started services to be healthy

internal/db/reset/reset.go

@@ -141,7 +141,7 @@ func RestartDatabase(ctx context.Context) {
 		fmt.Fprintln(os.Stderr, "Failed to restart database:", err)
 		return
 	}
-	if !WaitForHealthyDatabase(ctx, healthTimeout) {
+	if !WaitForHealthyService(ctx, utils.DbId, healthTimeout) {
 		fmt.Fprintln(os.Stderr, "Database is not healthy.")
 		return
 	}
@@ -151,17 +151,27 @@ func RestartDatabase(ctx context.Context) {
 	}
 }
 
-func WaitForHealthyDatabase(ctx context.Context, timeout time.Duration) bool {
-	// Poll for container health status
+func RetryEverySecond(callback func() bool, timeout time.Duration) bool {
 	now := time.Now()
 	expiry := now.Add(timeout)
 	ticker := time.NewTicker(time.Second)
 	defer ticker.Stop()
 	for t := now; t.Before(expiry); t = <-ticker.C {
-		if resp, err := utils.Docker.ContainerInspect(ctx, utils.DbId); err == nil &&
-			resp.State.Health != nil && resp.State.Health.Status == "healthy" {
+		if callback() {
 			return true
 		}
 	}
 	return false
 }
+
+func IsContainerHealthy(ctx context.Context, container string) bool {
+	resp, err := utils.Docker.ContainerInspect(ctx, container)
+	return err == nil && resp.State.Health != nil && resp.State.Health.Status == "healthy"
+}
+
+func WaitForHealthyService(ctx context.Context, container string, timeout time.Duration) bool {
+	probe := func() bool {
+		return IsContainerHealthy(ctx, container)
+	}
+	return RetryEverySecond(probe, timeout)
+}

internal/db/start/start.go

@@ -81,7 +81,7 @@ func StartDatabase(ctx context.Context, fsys afero.Fs, w io.Writer, options ...f
 }
 
 func initDatabase(ctx context.Context, fsys afero.Fs, w io.Writer, options ...func(*pgx.ConnConfig)) error {
-	if !reset.WaitForHealthyDatabase(ctx, 20*time.Second) {
+	if !reset.WaitForHealthyService(ctx, utils.DbId, 20*time.Second) {
 		fmt.Fprintln(os.Stderr, "Database is not healthy.")
 	}
 	// Initialise globals

internal/start/start.go

@@ -6,15 +6,18 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"strconv"
 	"strings"
 	"text/template"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/go-connections/nat"
 	"github.com/jackc/pgx/v4"
 	"github.com/spf13/afero"
+	"github.com/supabase/cli/internal/db/reset"
 	"github.com/supabase/cli/internal/db/start"
 	"github.com/supabase/cli/internal/utils"
 )
@@ -91,6 +94,7 @@ func run(p utils.Program, ctx context.Context, fsys afero.Fs, excludedContainers
 	}
 
 	p.Send(utils.StatusMsg("Starting containers..."))
+	var started []string
 
 	// Start Kong.
 	if !isContainerExcluded(utils.KongImage, excluded) {
@@ -131,6 +135,7 @@ EOF
 		); err != nil {
 			return err
 		}
+		started = append(started, utils.KongId)
 	}
 
 	// Start GoTrue.
@@ -202,6 +207,12 @@ EOF
 			container.Config{
 				Image: utils.GotrueImage,
 				Env:   env,
+				Healthcheck: &container.HealthConfig{
+					Test:     []string{"CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9999/health"},
+					Interval: 2 * time.Second,
+					Timeout:  2 * time.Second,
+					Retries:  10,
+				},
 			},
 			container.HostConfig{
 				RestartPolicy: container.RestartPolicy{Name: "always"},
@@ -210,6 +221,7 @@ EOF
 		); err != nil {
 			return err
 		}
+		started = append(started, utils.GotrueId)
 	}
 
 	// Start Inbucket.
@@ -234,6 +246,7 @@ EOF
 		); err != nil {
 			return err
 		}
+		started = append(started, utils.InbucketId)
 	}
 
 	// Start Realtime.
@@ -263,6 +276,12 @@ EOF
 					"/bin/sh", "-c",
 					"/app/bin/migrate && /app/bin/realtime eval 'Realtime.Release.seeds(Realtime.Repo)' && /app/bin/server",
 				},
+				Healthcheck: &container.HealthConfig{
+					Test:     []string{"CMD", "printf", "\\0", ">", "/dev/tcp/localhost/4000"},
+					Interval: 2 * time.Second,
+					Timeout:  2 * time.Second,
+					Retries:  10,
+				},
 			},
 			container.HostConfig{
 				RestartPolicy: container.RestartPolicy{Name: "always"},
@@ -271,6 +290,7 @@ EOF
 		); err != nil {
 			return err
 		}
+		started = append(started, utils.RealtimeId)
 	}
 
 	// Start PostgREST.
@@ -294,6 +314,7 @@ EOF
 		); err != nil {
 			return err
 		}
+		started = append(started, utils.RestId)
 	}
 
 	// Start Storage.
@@ -321,6 +342,12 @@ EOF
 					"ENABLE_IMAGE_TRANSFORMATION=true",
 					"IMGPROXY_URL=http://" + utils.ImgProxyId + ":5001",
 				},
+				Healthcheck: &container.HealthConfig{
+					Test:     []string{"CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:5000/status"},
+					Interval: 2 * time.Second,
+					Timeout:  2 * time.Second,
+					Retries:  10,
+				},
 			},
 			container.HostConfig{
 				RestartPolicy: container.RestartPolicy{Name: "always"},
@@ -329,6 +356,7 @@ EOF
 		); err != nil {
 			return err
 		}
+		started = append(started, utils.StorageId)
 	}
 
 	// Start Storage ImgProxy.
@@ -342,6 +370,12 @@ EOF
 					"IMGPROXY_LOCAL_FILESYSTEM_ROOT=/",
 					"IMGPROXY_USE_ETAG=/",
 				},
+				Healthcheck: &container.HealthConfig{
+					Test:     []string{"CMD", "imgproxy", "health"},
+					Interval: 2 * time.Second,
+					Timeout:  2 * time.Second,
+					Retries:  10,
+				},
 			},
 			container.HostConfig{
 				VolumesFrom:   []string{utils.StorageId},
@@ -351,6 +385,7 @@ EOF
 		); err != nil {
 			return err
 		}
+		started = append(started, utils.ImgProxyId)
 	}
 
 	// Start pg-meta.
@@ -363,6 +398,12 @@ EOF
 					"PG_META_PORT=8080",
 					"PG_META_DB_HOST=" + utils.DbId,
 				},
+				Healthcheck: &container.HealthConfig{
+					Test:     []string{"CMD", "node", "-e", "require('http').get('http://localhost:8080/health', (r) => {if (r.statusCode !== 200) throw new Error(r.statusCode)})"},
+					Interval: 2 * time.Second,
+					Timeout:  2 * time.Second,
+					Retries:  10,
+				},
 			},
 			container.HostConfig{
 				RestartPolicy: container.RestartPolicy{Name: "always"},
@@ -371,6 +412,7 @@ EOF
 		); err != nil {
 			return err
 		}
+		started = append(started, utils.PgmetaId)
 	}
 
 	// Start Studio.
@@ -389,6 +431,12 @@ EOF
 					"SUPABASE_ANON_KEY=" + utils.AnonKey,
 					"SUPABASE_SERVICE_KEY=" + utils.ServiceRoleKey,
 				},
+				Healthcheck: &container.HealthConfig{
+					Test:     []string{"CMD", "node", "-e", "require('http').get('http://localhost:3000/api/profile', (r) => {if (r.statusCode !== 200) throw new Error(r.statusCode)})"},
+					Interval: 2 * time.Second,
+					Timeout:  2 * time.Second,
+					Retries:  10,
+				},
 			},
 			container.HostConfig{
 				PortBindings:  nat.PortMap{"3000/tcp": []nat.PortBinding{{HostPort: strconv.FormatUint(uint64(utils.Config.Studio.Port), 10)}}},
@@ -398,9 +446,10 @@ EOF
 		); err != nil {
 			return err
 		}
+		started = append(started, utils.StudioId)
 	}
 
-	return nil
+	return waitForServiceReady(ctx, started)
 }
 
 func isContainerExcluded(imageName string, excluded map[string]bool) bool {
@@ -418,3 +467,39 @@ func ExcludableContainers() []string {
 	}
 	return names
 }
+
+func waitForServiceReady(ctx context.Context, started []string) error {
+	probe := func() bool {
+		var unhealthy []string
+		for _, container := range started {
+			if !isServiceReady(ctx, container) {
+				unhealthy = append(unhealthy, container)
+			}
+		}
+		started = unhealthy
+		return len(started) == 0
+	}
+	if !reset.RetryEverySecond(probe, 10*time.Second) {
+		return fmt.Errorf("service not healthy: %v", started)
+	}
+	return nil
+}
+
+func isServiceReady(ctx context.Context, container string) bool {
+	if container == utils.RestId {
+		return IsPostgRESTHealthy(ctx)
+	}
+	return reset.IsContainerHealthy(ctx, container)
+}
+
+func IsPostgRESTHealthy(ctx context.Context) bool {
+	// PostgREST does not support native health checks
+	restUrl := fmt.Sprintf("http://localhost:%d/rest/v1/", utils.Config.Api.Port)
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, restUrl, nil)
+	if err != nil {
+		return false
+	}
+	req.Header.Add("apikey", utils.AnonKey)
+	resp, err := http.DefaultClient.Do(req)
+	return err == nil && resp.StatusCode == http.StatusOK
+}

internal/start/start_test.go

@@ -144,12 +144,6 @@ func TestDatabaseStart(t *testing.T) {
 		utils.DbId = "test-postgres"
 		utils.Config.Db.Port = 54322
 		apitest.MockDockerStart(utils.Docker, imageUrl, utils.DbId)
-		gock.New(utils.Docker.DaemonHost()).
-			Get("/v" + utils.Docker.ClientVersion() + "/containers/" + utils.DbId + "/json").
-			Reply(http.StatusOK).
-			JSON(types.ContainerJSON{ContainerJSONBase: &types.ContainerJSONBase{
-				State: &types.ContainerState{Health: &types.Health{Status: "healthy"}},
-			}})
 		// Start services
 		utils.KongId = "test-kong"
 		apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(utils.KongImage), utils.KongId)
@@ -168,7 +162,7 @@ func TestDatabaseStart(t *testing.T) {
 		apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(utils.PostgrestImage), utils.RestId)
 		utils.StorageId = "test-storage"
 		apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(utils.StorageImage), utils.StorageId)
-		utils.StudioId = "test-imgproxy"
+		utils.ImgProxyId = "test-imgproxy"
 		apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(utils.ImageProxyImage), utils.ImgProxyId)
 		utils.DifferId = "test-differ"
 		apitest.MockDockerStart(utils.Docker, utils.GetRegistryImageUrl(utils.DifferImage), utils.DifferId)
@@ -185,6 +179,22 @@ func TestDatabaseStart(t *testing.T) {
 			Reply("CREATE SCHEMA").
 			Query(utils.InitialSchemaSql).
 			Reply("CREATE SCHEMA")
+		// Setup health probes
+		started := []string{
+			utils.DbId, utils.KongId, utils.GotrueId, utils.InbucketId, utils.RealtimeId,
+			utils.StorageId, utils.ImgProxyId, utils.PgmetaId, utils.StudioId,
+		}
+		for _, container := range started {
+			gock.New(utils.Docker.DaemonHost()).
+				Get("/v" + utils.Docker.ClientVersion() + "/containers/" + container + "/json").
+				Reply(http.StatusOK).
+				JSON(types.ContainerJSON{ContainerJSONBase: &types.ContainerJSONBase{
+					State: &types.ContainerState{Health: &types.Health{Status: "healthy"}},
+				}})
+		}
+		gock.New("localhost").
+			Head("/rest/v1/").
+			Reply(http.StatusOK)
 		// Run test
 		err := utils.RunProgram(context.Background(), func(p utils.Program, ctx context.Context) error {
 			return run(p, context.Background(), fsys, []string{}, conn.Intercept)