From ca9b4aafc86af0f6fc086b1e1cc5d8e2a6e26105 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Tue, 24 May 2022 18:55:19 +0100 Subject: [PATCH 1/8] Split out function to wait until a homeserver deployment is ready to serve requests Signed-off-by: Sean Quah --- internal/docker/deployer.go | 121 +++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/internal/docker/deployer.go b/internal/docker/deployer.go index 6cde3c5d..85fb5973 100644 --- a/internal/docker/deployer.go +++ b/internal/docker/deployer.go @@ -329,61 +329,6 @@ func deployImage( ) } - var lastErr error - - // Inspect health status of container to check it is up - stopTime := time.Now().Add(cfg.SpawnHSTimeout) - iterCount := 0 - if inspect.State.Health != nil { - // If the container has a healthcheck, wait for it first - for { - iterCount += 1 - if time.Now().After(stopTime) { - lastErr = fmt.Errorf("timed out checking for homeserver to be up: %s", lastErr) - break - } - inspect, err = docker.ContainerInspect(ctx, containerID) - if err != nil { - lastErr = fmt.Errorf("inspect container %s => error: %s", containerID, err) - time.Sleep(50 * time.Millisecond) - continue - } - if inspect.State.Health.Status != "healthy" { - lastErr = fmt.Errorf("inspect container %s => health: %s", containerID, inspect.State.Health.Status) - time.Sleep(50 * time.Millisecond) - continue - } - lastErr = nil - break - - } - } - - // Having optionally waited for container to self-report healthy - // hit /versions to check it is actually responding - versionsURL := fmt.Sprintf("%s/_matrix/client/versions", baseURL) - - for { - iterCount += 1 - if time.Now().After(stopTime) { - lastErr = fmt.Errorf("timed out checking for homeserver to be up: %s", lastErr) - break - } - res, err := http.Get(versionsURL) - if err != nil { - lastErr = fmt.Errorf("GET %s => error: %s", versionsURL, err) - time.Sleep(50 * time.Millisecond) - continue - } - if res.StatusCode != 200 { - lastErr = fmt.Errorf("GET %s => HTTP %s", versionsURL, res.Status) - time.Sleep(50 * time.Millisecond) - continue - } - lastErr = nil - break - } - d := &HomeserverDeployment{ BaseURL: baseURL, FedBaseURL: fedBaseURL, @@ -392,8 +337,11 @@ func deployImage( ApplicationServices: asIDToRegistrationFromLabels(inspect.Config.Labels), DeviceIDs: deviceIDsFromLabels(inspect.Config.Labels), } - if lastErr != nil { - return d, fmt.Errorf("%s: failed to check server is up. %w", contextStr, lastErr) + + stopTime := time.Now().Add(cfg.SpawnHSTimeout) + iterCount, err := waitForContainer(ctx, docker, d, stopTime) + if err != nil { + return d, fmt.Errorf("%s: failed to check server is up. %w", contextStr, err) } else { if cfg.DebugLoggingEnabled { log.Printf("%s: Server is responding after %d iterations", contextStr, iterCount) @@ -428,6 +376,65 @@ func copyToContainer(docker *client.Client, containerID, path string, data []byt return nil } +// Waits until a homeserver deployment is ready to serve requests. +func waitForContainer(ctx context.Context, docker *client.Client, hsDep *HomeserverDeployment, stopTime time.Time) (iterCount int, err error) { + var lastErr error = nil + + iterCount = 0 + + // If the container has a healthcheck, wait for it first + for { + iterCount += 1 + if time.Now().After(stopTime) { + lastErr = fmt.Errorf("timed out checking for homeserver to be up: %s", lastErr) + break + } + inspect, err := docker.ContainerInspect(ctx, hsDep.ContainerID) + if err != nil { + lastErr = fmt.Errorf("inspect container %s => error: %s", hsDep.ContainerID, err) + time.Sleep(50 * time.Millisecond) + continue + } + if inspect.State.Health != nil && + inspect.State.Health.Status != "healthy" { + lastErr = fmt.Errorf("inspect container %s => health: %s", hsDep.ContainerID, inspect.State.Health.Status) + time.Sleep(50 * time.Millisecond) + continue + } + + // The container is healthy or has no health check. + lastErr = nil + break + } + + // Having optionally waited for container to self-report healthy + // hit /versions to check it is actually responding + versionsURL := fmt.Sprintf("%s/_matrix/client/versions", hsDep.BaseURL) + + for { + iterCount += 1 + if time.Now().After(stopTime) { + lastErr = fmt.Errorf("timed out checking for homeserver to be up: %s", lastErr) + break + } + res, err := http.Get(versionsURL) + if err != nil { + lastErr = fmt.Errorf("GET %s => error: %s", versionsURL, err) + time.Sleep(50 * time.Millisecond) + continue + } + if res.StatusCode != 200 { + lastErr = fmt.Errorf("GET %s => HTTP %s", versionsURL, res.Status) + time.Sleep(50 * time.Millisecond) + continue + } + lastErr = nil + break + } + + return iterCount, lastErr +} + // RoundTripper is a round tripper that maps https://hs1 to the federation port of the container // e.g https://localhost:35352 type RoundTripper struct { From 0d2355c19e9abd097615d459a66a917e6450579c Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Tue, 24 May 2022 18:56:42 +0100 Subject: [PATCH 2/8] Add a function to restart a deployment Signed-off-by: Sean Quah --- internal/docker/deployer.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/internal/docker/deployer.go b/internal/docker/deployer.go index 85fb5973..05621e01 100644 --- a/internal/docker/deployer.go +++ b/internal/docker/deployer.go @@ -435,6 +435,40 @@ func waitForContainer(ctx context.Context, docker *client.Client, hsDep *Homeser return iterCount, lastErr } +// Restart a deployment. +func (dep *Deployment) Restart() error { + ctx := context.Background() + + for _, hsDep := range dep.HS { + err := dep.Deployer.Docker.ContainerStop(ctx, hsDep.ContainerID, &dep.Config.SpawnHSTimeout) + if err != nil { + return fmt.Errorf("failed to restart container %s: %s", hsDep.ContainerID, err) + } + + // Remove the container from the network. If we don't do this, + // (re)starting the container fails with an error like + // "Error response from daemon: endpoint with name complement_fed_1_fed.alice.hs1_1 already exists in network complement_fed_alice". + err = dep.Deployer.Docker.NetworkDisconnect(ctx, dep.Deployer.networkID, hsDep.ContainerID, false) + if err != nil { + return fmt.Errorf("failed to restart container %s: %s", hsDep.ContainerID, err) + } + + err = dep.Deployer.Docker.ContainerStart(ctx, hsDep.ContainerID, types.ContainerStartOptions{}) + if err != nil { + return fmt.Errorf("failed to restart container %s: %s", hsDep.ContainerID, err) + } + + // Wait for the container to be ready. + stopTime := time.Now().Add(dep.Config.SpawnHSTimeout) + _, err = waitForContainer(ctx, dep.Deployer.Docker, &hsDep, stopTime) + if err != nil { + return fmt.Errorf("failed to restart container %s: %s", hsDep.ContainerID, err) + } + } + + return nil +} + // RoundTripper is a round tripper that maps https://hs1 to the federation port of the container // e.g https://localhost:35352 type RoundTripper struct { From 7c3dc41338b95217c3d9c4871a0cd0dbf4ccecf3 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Tue, 14 Jun 2022 15:21:22 +0100 Subject: [PATCH 3/8] Factor out `waitForPorts` function, which returns a container's baseURL and fedBaseURL --- internal/docker/deployer.go | 45 ++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/internal/docker/deployer.go b/internal/docker/deployer.go index 05621e01..2eed920c 100644 --- a/internal/docker/deployer.go +++ b/internal/docker/deployer.go @@ -301,27 +301,14 @@ func deployImage( log.Printf("%s: Started container %s", contextStr, containerID) } - // We need to hammer the inspect endpoint until the ports show up, they don't appear immediately. - var inspect types.ContainerJSON - var baseURL, fedBaseURL string - inspectStartTime := time.Now() - for time.Since(inspectStartTime) < time.Second { - inspect, err = docker.ContainerInspect(ctx, containerID) - if err != nil { - return stubDeployment, err - } - if inspect.State != nil && !inspect.State.Running { - // the container exited, bail out with a container ID for logs - return stubDeployment, fmt.Errorf("container is not running, state=%v", inspect.State.Status) - } - baseURL, fedBaseURL, err = endpoints(inspect.NetworkSettings.Ports, 8008, 8448) - if err == nil { - break - } - } + baseURL, fedBaseURL, err := waitForPorts(ctx, docker, containerID) if err != nil { return stubDeployment, fmt.Errorf("%s : image %s : %w", contextStr, imageID, err) } + inspect, err := docker.ContainerInspect(ctx, containerID) + if err != nil { + return stubDeployment, err + } for vol := range inspect.Config.Volumes { log.Printf( "WARNING: %s has a named VOLUME %s - volumes can lead to unpredictable behaviour due to "+ @@ -376,6 +363,28 @@ func copyToContainer(docker *client.Client, containerID, path string, data []byt return nil } +// Waits until a homeserver container has NAT ports assigned and returns its clientside API URL and federation API URL. +func waitForPorts(ctx context.Context, docker *client.Client, containerID string) (baseURL string, fedBaseURL string, err error) { + // We need to hammer the inspect endpoint until the ports show up, they don't appear immediately. + var inspect types.ContainerJSON + inspectStartTime := time.Now() + for time.Since(inspectStartTime) < time.Second { + inspect, err = docker.ContainerInspect(ctx, containerID) + if err != nil { + return "", "", err + } + if inspect.State != nil && !inspect.State.Running { + // the container exited, bail out with a container ID for logs + return "", "", fmt.Errorf("container is not running, state=%v", inspect.State.Status) + } + baseURL, fedBaseURL, err = endpoints(inspect.NetworkSettings.Ports, 8008, 8448) + if err == nil { + break + } + } + return baseURL, fedBaseURL, nil +} + // Waits until a homeserver deployment is ready to serve requests. func waitForContainer(ctx context.Context, docker *client.Client, hsDep *HomeserverDeployment, stopTime time.Time) (iterCount int, err error) { var lastErr error = nil From a8d45a9f55c200924e518f5912851d3dc0188e18 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Tue, 14 Jun 2022 15:25:32 +0100 Subject: [PATCH 4/8] Refer to `HomeserverDeployment`s by pointer, since their fields may change --- cmd/homerunner/route_create.go | 4 ++-- internal/docker/deployer.go | 6 +++--- internal/docker/deployment.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/homerunner/route_create.go b/cmd/homerunner/route_create.go index dc7679be..01896e75 100644 --- a/cmd/homerunner/route_create.go +++ b/cmd/homerunner/route_create.go @@ -17,8 +17,8 @@ type ReqCreate struct { } type ResCreate struct { - Homeservers map[string]docker.HomeserverDeployment `json:"homeservers"` - Expires time.Time `json:"expires"` + Homeservers map[string]*docker.HomeserverDeployment `json:"homeservers"` + Expires time.Time `json:"expires"` } // RouteCreate handles creating blueprint deployments. There are 3 supported types of requests: diff --git a/internal/docker/deployer.go b/internal/docker/deployer.go index 2eed920c..b657d8b3 100644 --- a/internal/docker/deployer.go +++ b/internal/docker/deployer.go @@ -78,7 +78,7 @@ func (d *Deployer) Deploy(ctx context.Context, blueprintName string) (*Deploymen dep := &Deployment{ Deployer: d, BlueprintName: blueprintName, - HS: make(map[string]HomeserverDeployment), + HS: make(map[string]*HomeserverDeployment), Config: d.config, } images, err := d.Docker.ImageList(ctx, types.ImageListOptions{ @@ -127,7 +127,7 @@ func (d *Deployer) Deploy(ctx context.Context, blueprintName string) (*Deploymen } mu.Lock() d.log("%s -> %s (%s)\n", contextStr, deployment.BaseURL, deployment.ContainerID) - dep.HS[hsName] = *deployment + dep.HS[hsName] = deployment mu.Unlock() return nil } @@ -469,7 +469,7 @@ func (dep *Deployment) Restart() error { // Wait for the container to be ready. stopTime := time.Now().Add(dep.Config.SpawnHSTimeout) - _, err = waitForContainer(ctx, dep.Deployer.Docker, &hsDep, stopTime) + _, err = waitForContainer(ctx, dep.Deployer.Docker, hsDep, stopTime) if err != nil { return fmt.Errorf("failed to restart container %s: %s", hsDep.ContainerID, err) } diff --git a/internal/docker/deployment.go b/internal/docker/deployment.go index b202e196..4b91d69c 100644 --- a/internal/docker/deployment.go +++ b/internal/docker/deployment.go @@ -16,7 +16,7 @@ type Deployment struct { // The name of the deployed blueprint BlueprintName string // A map of HS name to a HomeserverDeployment - HS map[string]HomeserverDeployment + HS map[string]*HomeserverDeployment Config *config.Complement } From 0ad7b08b1f119f56f2e97e215638ee0d5010fe38 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Tue, 14 Jun 2022 15:33:12 +0100 Subject: [PATCH 5/8] Track `client.CSAPI` instances for each `HomeserverDeployment` --- internal/docker/deployment.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/docker/deployment.go b/internal/docker/deployment.go index 4b91d69c..1a0c52a6 100644 --- a/internal/docker/deployment.go +++ b/internal/docker/deployment.go @@ -28,6 +28,7 @@ type HomeserverDeployment struct { AccessTokens map[string]string // e.g { "@alice:hs1": "myAcc3ssT0ken" } ApplicationServices map[string]string // e.g { "my-as-id": "id: xxx\nas_token: xxx ..."} } DeviceIDs map[string]string // e.g { "@alice:hs1": "myDeviceID" } + CSAPIClients []*client.CSAPI } // Destroy the entire deployment. Destroys all running containers. If `printServerLogs` is true, @@ -56,7 +57,7 @@ func (d *Deployment) Client(t *testing.T, hsName, userID string) *client.CSAPI { if deviceID == "" && userID != "" { t.Logf("WARNING: Deployment.Client - HS name '%s' - user ID '%s' - deviceID not found", hsName, userID) } - return &client.CSAPI{ + client := &client.CSAPI{ UserID: userID, AccessToken: token, DeviceID: deviceID, @@ -65,6 +66,8 @@ func (d *Deployment) Client(t *testing.T, hsName, userID string) *client.CSAPI { SyncUntilTimeout: 5 * time.Second, Debug: d.Deployer.debugLogging, } + dep.CSAPIClients = append(dep.CSAPIClients, client) + return client } // RegisterUser within a homeserver and return an authenticatedClient, Fails the test if the hsName is not found. @@ -81,6 +84,7 @@ func (d *Deployment) RegisterUser(t *testing.T, hsName, localpart, password stri SyncUntilTimeout: 5 * time.Second, Debug: d.Deployer.debugLogging, } + dep.CSAPIClients = append(dep.CSAPIClients, client) var userID, accessToken, deviceID string if isAdmin { userID, accessToken, deviceID = client.RegisterSharedSecret(t, localpart, password, isAdmin) From e5ca691e7d6b9349ce97d68f48c4c68c93f3460f Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Tue, 14 Jun 2022 15:00:36 +0100 Subject: [PATCH 6/8] Move `Deployment.Restart` to the correct file and split out Docker usage back into `deployer.go` --- internal/docker/deployer.go | 65 +++++++++++++++++------------------ internal/docker/deployment.go | 14 ++++++++ 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/internal/docker/deployer.go b/internal/docker/deployer.go index b657d8b3..36584319 100644 --- a/internal/docker/deployer.go +++ b/internal/docker/deployer.go @@ -176,6 +176,37 @@ func (d *Deployer) Destroy(dep *Deployment, printServerLogs bool) { } } +// Restart a homeserver deployment. +func (d *Deployer) Restart(hsDep *HomeserverDeployment, cfg *config.Complement) error { + ctx := context.Background() + err := d.Docker.ContainerStop(ctx, hsDep.ContainerID, &cfg.SpawnHSTimeout) + if err != nil { + return fmt.Errorf("Restart: Failed to restart container %s: %s", hsDep.ContainerID, err) + } + + // Remove the container from the network. If we don't do this, + // (re)starting the container fails with an error like + // "Error response from daemon: endpoint with name complement_fed_1_fed.alice.hs1_1 already exists in network complement_fed_alice". + err = d.Docker.NetworkDisconnect(ctx, d.networkID, hsDep.ContainerID, false) + if err != nil { + return fmt.Errorf("Restart: Failed to restart container %s: %s", hsDep.ContainerID, err) + } + + err = d.Docker.ContainerStart(ctx, hsDep.ContainerID, types.ContainerStartOptions{}) + if err != nil { + return fmt.Errorf("Restart: Failed to restart container %s: %s", hsDep.ContainerID, err) + } + + // Wait for the container to be ready. + stopTime := time.Now().Add(cfg.SpawnHSTimeout) + _, err = waitForContainer(ctx, d.Docker, hsDep, stopTime) + if err != nil { + return fmt.Errorf("Restart: Failed to restart container %s: %s", hsDep.ContainerID, err) + } + + return nil +} + // nolint func deployImage( docker *client.Client, imageID string, containerName, pkgNamespace, blueprintName, hsName string, @@ -444,40 +475,6 @@ func waitForContainer(ctx context.Context, docker *client.Client, hsDep *Homeser return iterCount, lastErr } -// Restart a deployment. -func (dep *Deployment) Restart() error { - ctx := context.Background() - - for _, hsDep := range dep.HS { - err := dep.Deployer.Docker.ContainerStop(ctx, hsDep.ContainerID, &dep.Config.SpawnHSTimeout) - if err != nil { - return fmt.Errorf("failed to restart container %s: %s", hsDep.ContainerID, err) - } - - // Remove the container from the network. If we don't do this, - // (re)starting the container fails with an error like - // "Error response from daemon: endpoint with name complement_fed_1_fed.alice.hs1_1 already exists in network complement_fed_alice". - err = dep.Deployer.Docker.NetworkDisconnect(ctx, dep.Deployer.networkID, hsDep.ContainerID, false) - if err != nil { - return fmt.Errorf("failed to restart container %s: %s", hsDep.ContainerID, err) - } - - err = dep.Deployer.Docker.ContainerStart(ctx, hsDep.ContainerID, types.ContainerStartOptions{}) - if err != nil { - return fmt.Errorf("failed to restart container %s: %s", hsDep.ContainerID, err) - } - - // Wait for the container to be ready. - stopTime := time.Now().Add(dep.Config.SpawnHSTimeout) - _, err = waitForContainer(ctx, dep.Deployer.Docker, hsDep, stopTime) - if err != nil { - return fmt.Errorf("failed to restart container %s: %s", hsDep.ContainerID, err) - } - } - - return nil -} - // RoundTripper is a round tripper that maps https://hs1 to the federation port of the container // e.g https://localhost:35352 type RoundTripper struct { diff --git a/internal/docker/deployment.go b/internal/docker/deployment.go index 1a0c52a6..18f5a640 100644 --- a/internal/docker/deployment.go +++ b/internal/docker/deployment.go @@ -100,3 +100,17 @@ func (d *Deployment) RegisterUser(t *testing.T, hsName, localpart, password stri client.DeviceID = deviceID return client } + +// Restart a deployment. +func (dep *Deployment) Restart(t *testing.T) error { + t.Helper() + for _, hsDep := range dep.HS { + err := dep.Deployer.Restart(hsDep, dep.Config) + if err != nil { + t.Errorf("Deployment.Restart: %s", err) + return err + } + } + + return nil +} From a3bdc87c335e19a4284882298b658f6a122451f6 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Tue, 14 Jun 2022 16:07:53 +0100 Subject: [PATCH 7/8] Update the `BaseURL`s of CSAPI clients when restarting a deployment --- internal/docker/deployer.go | 6 ++++++ internal/docker/deployment.go | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/internal/docker/deployer.go b/internal/docker/deployer.go index 36584319..602e6f76 100644 --- a/internal/docker/deployer.go +++ b/internal/docker/deployer.go @@ -198,6 +198,12 @@ func (d *Deployer) Restart(hsDep *HomeserverDeployment, cfg *config.Complement) } // Wait for the container to be ready. + baseURL, fedBaseURL, err := waitForPorts(ctx, d.Docker, hsDep.ContainerID) + if err != nil { + return fmt.Errorf("Restart: Failed to restart container %s: %s", hsDep.ContainerID, err) + } + hsDep.SetEndpoints(baseURL, fedBaseURL) + stopTime := time.Now().Add(cfg.SpawnHSTimeout) _, err = waitForContainer(ctx, d.Docker, hsDep, stopTime) if err != nil { diff --git a/internal/docker/deployment.go b/internal/docker/deployment.go index 18f5a640..e435006d 100644 --- a/internal/docker/deployment.go +++ b/internal/docker/deployment.go @@ -31,6 +31,16 @@ type HomeserverDeployment struct { CSAPIClients []*client.CSAPI } +// Updates the client and federation base URLs of the homeserver deployment. +func (hsDep *HomeserverDeployment) SetEndpoints(baseURL string, fedBaseURL string) { + hsDep.BaseURL = baseURL + hsDep.FedBaseURL = fedBaseURL + + for _, client := range hsDep.CSAPIClients { + client.BaseURL = baseURL + } +} + // Destroy the entire deployment. Destroys all running containers. If `printServerLogs` is true, // will print container logs before killing the container. func (d *Deployment) Destroy(t *testing.T) { From cf3641508bb35ce53062d0edbae6d6f17ab20d35 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Thu, 16 Jun 2022 13:27:24 +0100 Subject: [PATCH 8/8] Apply suggestions from code review Co-authored-by: kegsay --- internal/docker/deployer.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/docker/deployer.go b/internal/docker/deployer.go index 602e6f76..d242d134 100644 --- a/internal/docker/deployer.go +++ b/internal/docker/deployer.go @@ -181,7 +181,7 @@ func (d *Deployer) Restart(hsDep *HomeserverDeployment, cfg *config.Complement) ctx := context.Background() err := d.Docker.ContainerStop(ctx, hsDep.ContainerID, &cfg.SpawnHSTimeout) if err != nil { - return fmt.Errorf("Restart: Failed to restart container %s: %s", hsDep.ContainerID, err) + return fmt.Errorf("Restart: Failed to stop container %s: %s", hsDep.ContainerID, err) } // Remove the container from the network. If we don't do this, @@ -189,18 +189,18 @@ func (d *Deployer) Restart(hsDep *HomeserverDeployment, cfg *config.Complement) // "Error response from daemon: endpoint with name complement_fed_1_fed.alice.hs1_1 already exists in network complement_fed_alice". err = d.Docker.NetworkDisconnect(ctx, d.networkID, hsDep.ContainerID, false) if err != nil { - return fmt.Errorf("Restart: Failed to restart container %s: %s", hsDep.ContainerID, err) + return fmt.Errorf("Restart: Failed to disconnect container %s: %s", hsDep.ContainerID, err) } err = d.Docker.ContainerStart(ctx, hsDep.ContainerID, types.ContainerStartOptions{}) if err != nil { - return fmt.Errorf("Restart: Failed to restart container %s: %s", hsDep.ContainerID, err) + return fmt.Errorf("Restart: Failed to start container %s: %s", hsDep.ContainerID, err) } // Wait for the container to be ready. baseURL, fedBaseURL, err := waitForPorts(ctx, d.Docker, hsDep.ContainerID) if err != nil { - return fmt.Errorf("Restart: Failed to restart container %s: %s", hsDep.ContainerID, err) + return fmt.Errorf("Restart: Failed to get ports for container %s: %s", hsDep.ContainerID, err) } hsDep.SetEndpoints(baseURL, fedBaseURL)