diff --git a/internal/pkg/aws/ecs/task.go b/internal/pkg/aws/ecs/task.go index b2b763037e7..4a982cf315f 100644 --- a/internal/pkg/aws/ecs/task.go +++ b/internal/pkg/aws/ecs/task.go @@ -55,7 +55,7 @@ type Task ecs.Task // becomes "4082490e (sample-fargate:2)" func (t Task) String() string { taskID, _ := TaskID(aws.StringValue(t.TaskArn)) - taskID = shortTaskID(taskID) + taskID = ShortTaskID(taskID) taskDefName, _ := taskDefinitionName(aws.StringValue(t.TaskDefinitionArn)) return fmt.Sprintf("%s (%s)", taskID, taskDefName) } @@ -291,7 +291,9 @@ func TaskDefinitionVersion(taskDefARN string) (int, error) { return version, nil } -func shortTaskID(id string) string { +// ShortTaskID truncates a string to a maximum length of shortTaskIDLength. +// If the input is shorter, it remains unchanged. +func ShortTaskID(id string) string { if len(id) >= shortTaskIDLength { return id[:shortTaskIDLength] } diff --git a/internal/pkg/aws/ecs/task_test.go b/internal/pkg/aws/ecs/task_test.go index cd23e0fa6a6..3bbd75e46f2 100644 --- a/internal/pkg/aws/ecs/task_test.go +++ b/internal/pkg/aws/ecs/task_test.go @@ -628,6 +628,35 @@ func TestTaskDefinition_EntryPoint(t *testing.T) { } } +func TestShortTaskID(t *testing.T) { + testCases := map[string]struct { + inTaskId string + wantedTaskId string + }{ + "return truncated short task id": { + inTaskId: "37930fffc2104a1db455aef109b5d122", + wantedTaskId: "37930fff", + }, + "return given short taskid": { + inTaskId: "37930fff", + wantedTaskId: "37930fff", + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + // GIVEN + ctrl := gomock.NewController(t) + defer ctrl.Finish() + // WHEN + got := ShortTaskID(tc.inTaskId) + // THEN + require.Equal(t, tc.wantedTaskId, got) + }) + + } +} + func TestFilterRunningTasks(t *testing.T) { testCases := map[string]struct { inTasks []*Task diff --git a/internal/pkg/deploy/cloudformation/cloudformation_test.go b/internal/pkg/deploy/cloudformation/cloudformation_test.go index 71c5ac03bd4..81f012f66c3 100644 --- a/internal/pkg/deploy/cloudformation/cloudformation_test.go +++ b/internal/pkg/deploy/cloudformation/cloudformation_test.go @@ -359,6 +359,7 @@ Resources: }, }, }, nil) + mockECS.EXPECT().StoppedServiceTasks("cluster", "service").Return(nil, nil) mockCFN.EXPECT().Describe(stackName).Return(&cloudformation.StackDescription{ StackStatus: aws.String("CREATE_COMPLETE"), }, nil) diff --git a/internal/pkg/deploy/cloudformation/mocks/mock_cloudformation.go b/internal/pkg/deploy/cloudformation/mocks/mock_cloudformation.go index 8dddc27ce5b..40ef169c9ff 100644 --- a/internal/pkg/deploy/cloudformation/mocks/mock_cloudformation.go +++ b/internal/pkg/deploy/cloudformation/mocks/mock_cloudformation.go @@ -189,6 +189,21 @@ func (mr *MockecsClientMockRecorder) Service(clusterName, serviceName interface{ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Service", reflect.TypeOf((*MockecsClient)(nil).Service), clusterName, serviceName) } +// StoppedServiceTasks mocks base method. +func (m *MockecsClient) StoppedServiceTasks(cluster, service string) ([]*ecs.Task, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StoppedServiceTasks", cluster, service) + ret0, _ := ret[0].([]*ecs.Task) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StoppedServiceTasks indicates an expected call of StoppedServiceTasks. +func (mr *MockecsClientMockRecorder) StoppedServiceTasks(cluster, service interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoppedServiceTasks", reflect.TypeOf((*MockecsClient)(nil).StoppedServiceTasks), cluster, service) +} + // MockcwClient is a mock of cwClient interface. type MockcwClient struct { ctrl *gomock.Controller diff --git a/internal/pkg/stream/ecs.go b/internal/pkg/stream/ecs.go index 52861640208..e3b3f9652e6 100644 --- a/internal/pkg/stream/ecs.go +++ b/internal/pkg/stream/ecs.go @@ -6,6 +6,7 @@ package stream import ( "fmt" "math/rand" + "sort" "strings" "sync" "time" @@ -24,11 +25,16 @@ const ( rollOutEmpty = "" ) +const ( + ecsScalingActivity = "Scaling activity initiated by" +) + var ecsEventFailureKeywords = []string{"fail", "unhealthy", "error", "throttle", "unable", "missing", "alarm detected", "rolling back"} // ECSServiceDescriber is the interface to describe an ECS service. type ECSServiceDescriber interface { Service(clusterName, serviceName string) (*ecs.Service, error) + StoppedServiceTasks(cluster, service string) ([]*ecs.Task, error) } // CloudWatchDescriber is the interface to describe CW alarms. @@ -47,6 +53,7 @@ type ECSDeployment struct { RolloutState string CreatedAt time.Time UpdatedAt time.Time + Id string } func (d ECSDeployment) isPrimary() bool { @@ -69,6 +76,7 @@ type ECSService struct { Deployments []ECSDeployment LatestFailureEvents []string Alarms []cloudwatch.AlarmStatus + StoppedTasks []ecs.Task } // ECSDeploymentStreamer is a Streamer for ECSService descriptions until the deployment is completed. @@ -135,6 +143,7 @@ func (s *ECSDeploymentStreamer) Fetch() (next time.Time, done bool, err error) { s.ecsRetries = 0 var deployments []ECSDeployment + var primaryDeploymentId string for _, deployment := range out.Deployments { status := aws.StringValue(deployment.Status) desiredCount, runningCount := aws.Int64Value(deployment.DesiredCount), aws.Int64Value(deployment.RunningCount) @@ -148,12 +157,44 @@ func (s *ECSDeploymentStreamer) Fetch() (next time.Time, done bool, err error) { RolloutState: aws.StringValue(deployment.RolloutState), CreatedAt: aws.TimeValue(deployment.CreatedAt), UpdatedAt: aws.TimeValue(deployment.UpdatedAt), + Id: aws.StringValue(deployment.Id), } deployments = append(deployments, rollingDeploy) if isDeploymentDone(rollingDeploy, s.deploymentCreationTime) { done = true } + if rollingDeploy.isPrimary() { + primaryDeploymentId = rollingDeploy.Id + } + } + stoppedSvcTasks, err := s.client.StoppedServiceTasks(s.cluster, s.service) + if err != nil { + if request.IsErrorThrottle(err) { + s.ecsRetries += 1 + return nextFetchDate(s.clock, s.rand, s.ecsRetries), false, nil + } + return next, false, fmt.Errorf("fetch stopped tasks: %w", err) + } + s.ecsRetries = 0 + + var stoppedTasks []ecs.Task + for _, st := range stoppedSvcTasks { + if stoppingAt := aws.TimeValue(st.StoppingAt); aws.StringValue(st.StartedBy) != primaryDeploymentId || stoppingAt.Before(s.deploymentCreationTime) || + (strings.Contains(aws.StringValue(st.StoppedReason), ecsScalingActivity)) { + continue + } + stoppedTasks = append(stoppedTasks, ecs.Task{ + TaskArn: st.TaskArn, + DesiredStatus: st.DesiredStatus, + LastStatus: st.LastStatus, + StoppedReason: st.StoppedReason, + StoppingAt: st.StoppingAt, + }) } + sort.SliceStable(stoppedTasks, func(i, j int) bool { + return aws.TimeValue(stoppedTasks[i].StoppingAt).After(aws.TimeValue(stoppedTasks[j].StoppingAt)) + }) + var failureMsgs []string for _, event := range out.Events { if createdAt := aws.TimeValue(event.CreatedAt); createdAt.Before(s.deploymentCreationTime) { @@ -187,6 +228,7 @@ func (s *ECSDeploymentStreamer) Fetch() (next time.Time, done bool, err error) { Deployments: deployments, LatestFailureEvents: failureMsgs, Alarms: alarms, + StoppedTasks: stoppedTasks, }) return nextFetchDate(s.clock, s.rand, 0), done, nil } diff --git a/internal/pkg/stream/ecs_test.go b/internal/pkg/stream/ecs_test.go index 017bf73bf93..571a348f733 100644 --- a/internal/pkg/stream/ecs_test.go +++ b/internal/pkg/stream/ecs_test.go @@ -5,10 +5,11 @@ package stream import ( "errors" - "github.com/aws/copilot-cli/internal/pkg/aws/cloudwatch" "testing" "time" + "github.com/aws/copilot-cli/internal/pkg/aws/cloudwatch" + "github.com/aws/aws-sdk-go/aws" awsecs "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/copilot-cli/internal/pkg/aws/ecs" @@ -16,8 +17,10 @@ import ( ) type mockECS struct { - out *ecs.Service - err error + out *ecs.Service + tasks []*ecs.Task + err error + taskError error } type mockCW struct { @@ -28,6 +31,9 @@ type mockCW struct { func (m mockECS) Service(clusterName, serviceName string) (*ecs.Service, error) { return m.out, m.err } +func (m mockECS) StoppedServiceTasks(clusterName, serviceName string) ([]*ecs.Task, error) { + return m.tasks, m.taskError +} func (m mockCW) AlarmStatuses(opts ...cloudwatch.DescribeAlarmOpts) ([]cloudwatch.AlarmStatus, error) { return m.out, m.err @@ -97,6 +103,37 @@ func TestECSDeploymentStreamer_Fetch(t *testing.T) { // THEN require.EqualError(t, err, "retrieve alarm statuses: some error") }) + t.Run("returns a wrapped error on stopped tasks call failure", func(t *testing.T) { + // GIVEN + m := mockECS{ + out: &ecs.Service{ + DeploymentConfiguration: &awsecs.DeploymentConfiguration{ + Alarms: &awsecs.DeploymentAlarms{ + AlarmNames: []*string{aws.String("alarm1"), aws.String("alarm2")}, + Enable: aws.Bool(true), + Rollback: aws.Bool(true), + }, + }, + }, + tasks: []*ecs.Task{ + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/testbugbash-testenv-Cluster-qrvEB"), + DesiredStatus: aws.String("Stopped"), + LastStatus: aws.String("Deprovisioning"), + StoppedReason: aws.String("unable to pull secrets"), + }, + }, + taskError: errors.New("some error"), + } + cw := mockCW{} + streamer := NewECSDeploymentStreamer(m, cw, "my-cluster", "my-svc", time.Now()) + + // WHEN + _, _, err := streamer.Fetch() + + // THEN + require.EqualError(t, err, "fetch stopped tasks: some error") + }) t.Run("stores events, alarms, and failures until deployment is done", func(t *testing.T) { // GIVEN oldStartDate := time.Date(2020, time.November, 23, 17, 0, 0, 0, time.UTC) @@ -113,6 +150,7 @@ func TestECSDeploymentStreamer_Fetch(t *testing.T) { Status: aws.String("PRIMARY"), TaskDefinition: aws.String("arn:aws:ecs:us-west-2:1111:task-definition/myapp-test-mysvc:2"), UpdatedAt: aws.Time(startDate), + Id: aws.String("ecs-svc/123"), }, { DesiredCount: aws.Int64(10), @@ -123,6 +161,7 @@ func TestECSDeploymentStreamer_Fetch(t *testing.T) { Status: aws.String("ACTIVE"), TaskDefinition: aws.String("arn:aws:ecs:us-west-2:1111:task-definition/myapp-test-mysvc:1"), UpdatedAt: aws.Time(oldStartDate), + Id: aws.String("ecs-svc/456"), }, }, DeploymentConfiguration: &awsecs.DeploymentConfiguration{ @@ -145,6 +184,40 @@ func TestECSDeploymentStreamer_Fetch(t *testing.T) { }, }, }, + tasks: []*ecs.Task{ + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEB"), + DesiredStatus: aws.String("Stopped"), + LastStatus: aws.String("Deprovisioning"), + StoppedReason: aws.String("unable to pull secrets"), + StoppingAt: aws.Time(startDate.Add(10 * time.Second)), + StartedBy: aws.String("ecs-svc/123"), + }, + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEBt"), + DesiredStatus: aws.String("Stopped"), + LastStatus: aws.String("Stopped"), + StoppedReason: aws.String("unable to pull secrets"), + StoppingAt: aws.Time(oldStartDate), + StartedBy: aws.String("ecs-svc/123"), + }, + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEBs"), + DesiredStatus: aws.String("Stopped"), + LastStatus: aws.String("Deprovisioning"), + StoppedReason: aws.String("ELB healthcheck failed"), + StoppingAt: aws.Time(startDate.Add(20 * time.Second)), + StartedBy: aws.String("ecs-svc/123"), + }, + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEBu"), + DesiredStatus: aws.String("Stopped"), + LastStatus: aws.String("Deprovisioning"), + StoppedReason: aws.String("Scaling activity initiated by deployment ecs-svc/mocktaskid"), + StoppingAt: aws.Time(startDate.Add(30 * time.Second)), + StartedBy: aws.String("ecs-svc/123"), + }, + }, } cw := mockCW{ out: []cloudwatch.AlarmStatus{ @@ -177,6 +250,7 @@ func TestECSDeploymentStreamer_Fetch(t *testing.T) { PendingCount: 0, RolloutState: "COMPLETED", UpdatedAt: startDate, + Id: "ecs-svc/123", }, { Status: "ACTIVE", @@ -187,6 +261,7 @@ func TestECSDeploymentStreamer_Fetch(t *testing.T) { PendingCount: 0, RolloutState: "FAILED", UpdatedAt: oldStartDate, + Id: "ecs-svc/456", }, }, Alarms: []cloudwatch.AlarmStatus{ @@ -200,6 +275,22 @@ func TestECSDeploymentStreamer_Fetch(t *testing.T) { }, }, LatestFailureEvents: []string{"deployment failed: alarm detected", "rolling back to deployment X"}, + StoppedTasks: []ecs.Task{ + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEBs"), + DesiredStatus: aws.String("Stopped"), + LastStatus: aws.String("Deprovisioning"), + StoppedReason: aws.String("ELB healthcheck failed"), + StoppingAt: aws.Time(startDate.Add(20 * time.Second)), + }, + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEB"), + DesiredStatus: aws.String("Stopped"), + LastStatus: aws.String("Deprovisioning"), + StoppedReason: aws.String("unable to pull secrets"), + StoppingAt: aws.Time(startDate.Add(10 * time.Second)), + }, + }, }, }, streamer.eventsToFlush) require.True(t, done, "there should be no more work to do since the deployment is completed") diff --git a/internal/pkg/term/progress/ecs.go b/internal/pkg/term/progress/ecs.go index be2c9447c6f..ea51f30bb5e 100644 --- a/internal/pkg/term/progress/ecs.go +++ b/internal/pkg/term/progress/ecs.go @@ -7,10 +7,15 @@ import ( "bytes" "fmt" "io" + "sort" "strconv" + "strings" "sync" + "time" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/copilot-cli/internal/pkg/aws/cloudwatch" + "github.com/dustin/go-humanize/english" "github.com/aws/copilot-cli/internal/pkg/aws/ecs" "github.com/aws/copilot-cli/internal/pkg/stream" @@ -21,6 +26,10 @@ const ( maxServiceEventsToDisplay = 5 // Total number of events we want to display at most for ECS service events. ) +const ( + maxStoppedTasksToDisplay = 2 +) + // ECSServiceSubscriber is the interface to subscribe channels to ECS service descriptions. type ECSServiceSubscriber interface { Subscribe() <-chan stream.ECSService @@ -40,9 +49,10 @@ func ListeningRollingUpdateRenderer(streamer ECSServiceSubscriber, opts RenderOp type rollingUpdateComponent struct { // Data to render. - deployments []stream.ECSDeployment - failureMsgs []string - alarms []cloudwatch.AlarmStatus + deployments []stream.ECSDeployment + failureMsgs []string + alarms []cloudwatch.AlarmStatus + stoppedTasks []ecs.Task // Style configuration for the component. padding int @@ -58,6 +68,10 @@ func (c *rollingUpdateComponent) Listen() { for ev := range c.stream { c.mu.Lock() c.deployments = ev.Deployments + c.stoppedTasks = ev.StoppedTasks + if len(c.stoppedTasks) > maxStoppedTasksToDisplay { + c.stoppedTasks = c.stoppedTasks[:maxStoppedTasksToDisplay] + } c.failureMsgs = append(c.failureMsgs, ev.LatestFailureEvents...) if len(c.failureMsgs) > c.maxLenFailureMsgs { c.failureMsgs = c.failureMsgs[len(c.failureMsgs)-c.maxLenFailureMsgs:] @@ -80,6 +94,12 @@ func (c *rollingUpdateComponent) Render(out io.Writer) (numLines int, err error) } numLines += nl + nl, err = c.renderStoppedTasks(buf) + if err != nil { + return 0, err + } + numLines += nl + nl, err = c.renderFailureMsgs(buf) if err != nil { return 0, err @@ -180,6 +200,96 @@ func (c *rollingUpdateComponent) renderAlarms(out io.Writer) (numLines int, err return renderComponents(out, components) } +func (c *rollingUpdateComponent) renderStoppedTasks(out io.Writer) (numLines int, err error) { + if len(c.stoppedTasks) == 0 { + return 0, nil + } + header := []string{"TaskId", "CurrentStatus", "DesiredStatus"} + var rows [][]string + title := fmt.Sprintf("Latest %d %s stopped reason", len(c.stoppedTasks), english.PluralWord(len(c.stoppedTasks), "task", "tasks")) + title = fmt.Sprintf("%s%s", color.DullRed.Sprintf("✘ "), color.Faint.Sprintf(title)) + childComponents := []Renderer{ + &singleLineComponent{}, // Add an empty line before rendering task stopped events. + &singleLineComponent{ + Text: title, + Padding: c.padding, + }, + } + type stoppedTasksInfo struct { + ids []string + latestStoppingAt time.Time + } + stopReason2Tasks := make(map[string]*stoppedTasksInfo, len(c.stoppedTasks)) + for _, st := range c.stoppedTasks { + id, err := ecs.TaskID(aws.StringValue(st.TaskArn)) + if err != nil { + return 0, err + } + tasks, ok := stopReason2Tasks[aws.StringValue(st.StoppedReason)] + if ok { + tasks.ids = append(tasks.ids, ecs.ShortTaskID(id)) + tasks.latestStoppingAt = aws.TimeValue(st.StoppingAt) + stopReason2Tasks[aws.StringValue(st.StoppedReason)] = tasks + } else { + stopReason2Tasks[aws.StringValue(st.StoppedReason)] = &stoppedTasksInfo{ + ids: []string{ecs.ShortTaskID(id)}, + latestStoppingAt: aws.TimeValue(st.StoppingAt), + } + } + rows = append(rows, []string{ + ecs.ShortTaskID(id), + aws.StringValue(st.LastStatus), + aws.StringValue(st.DesiredStatus), + }) + } + var sortedReasons []string + for reason := range stopReason2Tasks { + sortedReasons = append(sortedReasons, reason) + } + sort.SliceStable(sortedReasons, func(i, j int) bool { + return stopReason2Tasks[sortedReasons[i]].latestStoppingAt.After(stopReason2Tasks[sortedReasons[j]].latestStoppingAt) + }) + for _, reason := range sortedReasons { + for i, truncatedReason := range splitByLength(fmt.Sprintf("[%s]: %s", strings.Join(stopReason2Tasks[reason].ids, ","), reason), maxCellLength) { + pretty := fmt.Sprintf(" %s", truncatedReason) + if i == 0 { + pretty = fmt.Sprintf("- %s", truncatedReason) + } + childComponents = append(childComponents, &singleLineComponent{ + Text: pretty, + Padding: c.padding + nestedComponentPadding, + }) + } + } + table := newTableComponent(color.Faint.Sprintf("Latest %d stopped %s", len(c.stoppedTasks), english.PluralWord(len(c.stoppedTasks), "task", "tasks")), header, rows) + table.Padding = c.padding + childComponents = append(childComponents, + &singleLineComponent{}, + &singleLineComponent{ + Text: color.Faint.Sprintf("Troubleshoot task stopped reason"), + Padding: c.padding, + }, + &singleLineComponent{ + Text: fmt.Sprintf("1. You can run %s to see the logs of the last stopped task.", + color.HighlightCode("copilot svc logs --previous")), + Padding: c.padding + nestedComponentPadding, + }, + &singleLineComponent{ + Text: fmt.Sprintf("2. You can visit this article %s.", + color.Emphasize("https://repost.aws/knowledge-center/ecs-task-stopped")), + Padding: c.padding + nestedComponentPadding, + }) + treeComponent := treeComponent{ + Root: table, + Children: childComponents, + } + nl, err := treeComponent.Render(out) + if err != nil { + return 0, fmt.Errorf("render deployments table: %w", err) + } + return nl, err +} + func reverseStrings(arr []string) []string { reversed := make([]string, len(arr)) copy(reversed, arr) diff --git a/internal/pkg/term/progress/ecs_test.go b/internal/pkg/term/progress/ecs_test.go index a3cc7ea9dbf..fdb253e7e3a 100644 --- a/internal/pkg/term/progress/ecs_test.go +++ b/internal/pkg/term/progress/ecs_test.go @@ -4,11 +4,17 @@ package progress import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" "github.com/aws/copilot-cli/internal/pkg/aws/cloudwatch" + "github.com/aws/copilot-cli/internal/pkg/aws/ecs" "github.com/aws/copilot-cli/internal/pkg/stream" + "github.com/aws/copilot-cli/internal/pkg/term/color" "github.com/stretchr/testify/require" - "strings" - "testing" ) func TestRollingUpdateComponent_Listen(t *testing.T) { @@ -76,10 +82,12 @@ func TestRollingUpdateComponent_Listen(t *testing.T) { } func TestRollingUpdateComponent_Render(t *testing.T) { + startDate := time.Date(2020, time.November, 23, 18, 0, 0, 0, time.UTC) testCases := map[string]struct { - inDeployments []stream.ECSDeployment - inFailureMsgs []string - inAlarms []cloudwatch.AlarmStatus + inDeployments []stream.ECSDeployment + inFailureMsgs []string + inAlarms []cloudwatch.AlarmStatus + inStoppedTasks []ecs.Task wantedNumLines int wantedOut string @@ -153,6 +161,103 @@ Alarms alarm2 [ALARM] `, }, + "should render stopped tasks and their statuses": { + inStoppedTasks: []ecs.Task{ + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEBaBlImsZ/21479dca3393490a9d95f27353186bf6"), + DesiredStatus: aws.String("STOPPED"), + LastStatus: aws.String("DEPROVISIONING"), + StoppedReason: aws.String("ELB healthcheck failed"), + StoppingAt: aws.Time(startDate.Add(20 * time.Second)), + }, + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEBaBlImsZ/2243bac3ca1d4b3a8c66888348cba2e1"), + DesiredStatus: aws.String("STOPPED"), + LastStatus: aws.String("STOPPING"), + StoppedReason: aws.String("unable to pull secrets"), + StoppingAt: aws.Time(startDate.Add(10 * time.Second)), + }, + }, + wantedNumLines: 12, + wantedOut: fmt.Sprintf(`Latest 2 stopped tasks + TaskId CurrentStatus DesiredStatus + 21479dca DEPROVISIONING STOPPED + 2243bac3 STOPPING STOPPED + +✘ Latest 2 tasks stopped reason + - [21479dca]: ELB healthcheck failed + - [2243bac3]: unable to pull secrets + +Troubleshoot task stopped reason + 1. You can run %s to see the logs of the last stopped task. + 2. You can visit this article https://repost.aws/knowledge-center/ecs-task-stopped. +`, color.HighlightCode("copilot svc logs --previous")), + }, + "render collapse taskids if task reasons are same": { + inStoppedTasks: []ecs.Task{ + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEBaBlImsZ/21479dca3393490a9d95f27353186bf6"), + DesiredStatus: aws.String("STOPPED"), + LastStatus: aws.String("DEPROVISIONING"), + StoppedReason: aws.String("Essential container in the task exited"), + StoppingAt: aws.Time(startDate.Add(20 * time.Second)), + }, + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEBaBlImsZ/2243bac3ca1d4b3a8c66888348cba2e1"), + DesiredStatus: aws.String("STOPPED"), + LastStatus: aws.String("STOPPING"), + StoppedReason: aws.String("Essential container in the task exited"), + StoppingAt: aws.Time(startDate.Add(10 * time.Second)), + }, + }, + wantedNumLines: 11, + wantedOut: fmt.Sprintf(`Latest 2 stopped tasks + TaskId CurrentStatus DesiredStatus + 21479dca DEPROVISIONING STOPPED + 2243bac3 STOPPING STOPPED + +✘ Latest 2 tasks stopped reason + - [21479dca,2243bac3]: Essential container in the task exited + +Troubleshoot task stopped reason + 1. You can run %s to see the logs of the last stopped task. + 2. You can visit this article https://repost.aws/knowledge-center/ecs-task-stopped. +`, color.HighlightCode("copilot svc logs --previous")), + }, + "should render stopped tasks and split long stopped reasons": { + inStoppedTasks: []ecs.Task{ + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEBaBlImsZ/21479dca3393490a9d95f27353186bf6"), + DesiredStatus: aws.String("STOPPED"), + LastStatus: aws.String("DEPROVISIONING"), + StoppedReason: aws.String("ELB healthcheck failed"), + StoppingAt: aws.Time(startDate.Add(20 * time.Second)), + }, + { + TaskArn: aws.String("arn:aws:ecs:us-east-2:197732814171:task/bugbash-test-Cluster-qrvEBaBlImsZ/2243bac3ca1d4b3a8c66888348cba2e1"), + DesiredStatus: aws.String("STOPPED"), + LastStatus: aws.String("STOPPING"), + StoppedReason: aws.String("ResourceInitializationError: unable to pull secrets or registry auth: execution resource retrieval failed: unable to retrieve secrets from ssm: service call has been retried 1 time(s)"), + StoppingAt: aws.Time(startDate.Add(10 * time.Second)), + }, + }, + wantedNumLines: 14, + wantedOut: fmt.Sprintf(`Latest 2 stopped tasks + TaskId CurrentStatus DesiredStatus + 21479dca DEPROVISIONING STOPPED + 2243bac3 STOPPING STOPPED + +✘ Latest 2 tasks stopped reason + - [21479dca]: ELB healthcheck failed + - [2243bac3]: ResourceInitializationError: unable to pull secrets or reg + istry auth: execution resource retrieval failed: unable to retrieve se + crets from ssm: service call has been retried 1 time(s) + +Troubleshoot task stopped reason + 1. You can run %s to see the logs of the last stopped task. + 2. You can visit this article https://repost.aws/knowledge-center/ecs-task-stopped. +`, color.HighlightCode("copilot svc logs --previous")), + }, } for name, tc := range testCases { @@ -160,9 +265,10 @@ Alarms // GIVEN buf := new(strings.Builder) c := &rollingUpdateComponent{ - deployments: tc.inDeployments, - failureMsgs: tc.inFailureMsgs, - alarms: tc.inAlarms, + deployments: tc.inDeployments, + failureMsgs: tc.inFailureMsgs, + alarms: tc.inAlarms, + stoppedTasks: tc.inStoppedTasks, } // WHEN