diff --git a/internal/olm/.DS_Store b/internal/olm/.DS_Store new file mode 100644 index 0000000000..ef82ecf21f Binary files /dev/null and b/internal/olm/.DS_Store differ diff --git a/internal/olm/client/client.go b/internal/olm/client/client.go index 51ba991f06..b904f0ecb0 100644 --- a/internal/olm/client/client.go +++ b/internal/olm/client/client.go @@ -48,6 +48,38 @@ var ErrOLMNotInstalled = errors.New("no existing installation found") var Scheme = scheme.Scheme +// custom error struct to capture deployment errors +// while verifying CSV installs. +type resourceError struct { + name string + issue string +} +type podError struct { + resourceError +} +type deploymentError struct { + resourceError + podErrs podErrors +} +type deploymentErrors []deploymentError +type podErrors []podError + +func (e deploymentErrors) Error() string { + var sb strings.Builder + for _, i := range e { + sb.WriteString(fmt.Sprintf("deployment %s has error: %s\n%s", i.name, i.issue, i.podErrs.Error())) + } + return sb.String() +} + +func (e podErrors) Error() string { + var sb strings.Builder + for _, i := range e { + sb.WriteString(fmt.Sprintf("\tpod %s has error: %s\n", i.name, i.issue)) + } + return sb.String() +} + func init() { if err := olmapiv1alpha1.AddToScheme(Scheme); err != nil { log.Fatalf("Failed to add OLM operator API v1alpha1 types to scheme: %v", err) @@ -226,17 +258,18 @@ func (c Client) DoCSVWait(ctx context.Context, key types.NamespacedName) error { err := wait.PollImmediateUntil(time.Second, csvPhaseSucceeded, ctx.Done()) if err != nil && errors.Is(err, context.DeadlineExceeded) { - if depCheckErr := c.printDeploymentErrors(ctx, key, csv); depCheckErr != nil { - return fmt.Errorf("error printing operator resource errors: %v %v", err, depCheckErr) + depCheckErr := c.checkDeploymentErrors(ctx, key, csv) + if depCheckErr != nil { + return depCheckErr } } return err } -// TODO(btenneti) Refactor function to collect errors into customized error and return. -// printDeploymentErrors function loops through deployment specs of a given CSV, and prints reason +// checkDeploymentErrors function loops through deployment specs of a given CSV, and prints reason // in case of failures, based on deployment condition. -func (c Client) printDeploymentErrors(ctx context.Context, key types.NamespacedName, csv olmapiv1alpha1.ClusterServiceVersion) error { +func (c Client) checkDeploymentErrors(ctx context.Context, key types.NamespacedName, csv olmapiv1alpha1.ClusterServiceVersion) error { + depErrs := deploymentErrors{} if key.Namespace == "" { return fmt.Errorf("no namespace provided to get deployment failures") } @@ -248,25 +281,40 @@ func (c Client) printDeploymentErrors(ctx context.Context, key types.NamespacedN } depSelectors := ds.Spec.Selector if err := c.KubeClient.Get(ctx, depKey, dep); err != nil { - log.Printf("error getting operator deployment %q: %v", ds.Name, err) + depErrs = append(depErrs, deploymentError{ + resourceError: resourceError{ + name: ds.Name, + issue: err.Error(), + }, + }) continue } for _, s := range dep.Status.Conditions { - if s.Type == appsv1.DeploymentAvailable && s.Status == corev1.ConditionFalse { - log.Printf("operator deployment %q not available: %s", ds.Name, s.Reason) - if err := c.printPodErrors(ctx, depSelectors, key); err != nil { - return err + if s.Type == appsv1.DeploymentAvailable && s.Status != corev1.ConditionTrue { + depErr := deploymentError{ + resourceError: resourceError{ + name: ds.Name, + issue: s.Reason, + }, + } + podErr := c.checkPodErrors(ctx, depSelectors, key) + podErrs := podErrors{} + if errors.As(podErr, &podErrs) { + depErr.podErrs = append(depErr.podErrs, podErrs...) + } else { + return podErr } + depErrs = append(depErrs, depErr) } } } - return nil + return depErrs } -// printPodErrors loops through pods, and prints pod errors if any. -func (c Client) printPodErrors(ctx context.Context, depSelectors *metav1.LabelSelector, key types.NamespacedName) error { +// checkPodErrors loops through pods, and returns pod errors if any. +func (c Client) checkPodErrors(ctx context.Context, depSelectors *metav1.LabelSelector, key types.NamespacedName) error { // loop through pods and return specific error message. - podErrors := make(map[string]string) + podErr := podErrors{} podList := &corev1.PodList{} podLabelSelectors, err := metav1.LabelSelectorAsSelector(depSelectors) if err != nil { @@ -280,18 +328,21 @@ func (c Client) printPodErrors(ctx context.Context, depSelectors *metav1.LabelSe return fmt.Errorf("error getting Pods: %v", err) } for _, p := range podList.Items { - if p.Status.Phase != corev1.PodSucceeded { - for _, cs := range p.Status.ContainerStatuses { - if !cs.Ready { - podErrors[p.Name] = cs.State.Waiting.Message + for _, cs := range p.Status.ContainerStatuses { + if !cs.Ready { + if cs.State.Waiting != nil { + containerName := p.Name + ":" + cs.Name + podErr = append(podErr, podError{ + resourceError{ + name: containerName, + issue: cs.State.Waiting.Message, + }, + }) } } } } - if len(podErrors) > 0 { - log.Printf("pod errors: %v\n", podErrors) - } - return nil + return podErr } // GetInstalledVersion returns the OLM version installed in the namespace informed. diff --git a/internal/olm/client/client_test.go b/internal/olm/client/client_test.go index 30225398b0..b5f0a170ed 100644 --- a/internal/olm/client/client_test.go +++ b/internal/olm/client/client_test.go @@ -31,165 +31,231 @@ import ( ) var _ = Describe("Client", func() { - Describe("printDeploymentErrors", func() { + Describe("checkDeploymentErrors", func() { var ( fakeClient client.Client + csv olmapiv1alpha1.ClusterServiceVersion ) BeforeEach(func() { - fakeClient = fake.NewFakeClient( - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-operator-controller-manager-8687c65f7d-kc44t", - Namespace: "test-operator-system", - Labels: map[string]string{ - "control-plane": "controller-manager", - }, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - ContainerStatuses: []corev1.ContainerStatus{ - { - Ready: false, - State: corev1.ContainerState{ - Waiting: &corev1.ContainerStateWaiting{ - Message: "back-off 5m0s restarting failed container)", - Reason: "CrashLoopBackOff", + csv = olmapiv1alpha1.ClusterServiceVersion{ + Spec: olmapiv1alpha1.ClusterServiceVersionSpec{ + DisplayName: "test-operator", + InstallStrategy: olmapiv1alpha1.NamedInstallStrategy{ + StrategySpec: olmapiv1alpha1.StrategyDetailsDeployment{ + DeploymentSpecs: []olmapiv1alpha1.StrategyDeploymentSpec{ + { + Name: "dummy-operator", + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "dummylabel": "dummyvalue", + }, + }, + }, + }, + + { + Name: "test-operator-controller-manager", + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "control-plane": "controller-manager", + }, + }, }, }, }, }, }, }, - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dummypod", - Namespace: "testns", - }, - }, - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-operator-controller-manager-jjj", - Namespace: "test-operator-system", - Labels: map[string]string{ - "control-plane": "controller-manager", - }, - }, - Status: corev1.PodStatus{ - ContainerStatuses: []corev1.ContainerStatus{ - { - Ready: true, + } + }) + Context("with a valid csv", func() { + It("check error string for pod errors", func() { + fakeClient = fake.NewFakeClient( + + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-kc44t", + Namespace: "test-operator-system", + Labels: map[string]string{ + "control-plane": "controller-manager", }, }, - }, - }, - - &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-operator-controller-manager", - Namespace: "test-operator-system", - Labels: map[string]string{ - "control-plane": "controller-manager", + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "container1", + Ready: false, + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Message: "back-off 5m0s restarting failed container", + }, + }, + }, + }, }, }, - Status: appsv1.DeploymentStatus{ - Conditions: []appsv1.DeploymentCondition{ - { - Type: "Available", - Status: "False", - Reason: "MinimumReplicasUnavailable", - }, - { - Type: "Progressing", - Status: "True", - Reason: "NewReplicaSetAvailable", + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-controller-manager", + Namespace: "test-operator-system", + Labels: map[string]string{ + "control-plane": "controller-manager", }, }, - }, - }, - &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dummy-operator", - Namespace: "dummy-operator-system", - }, - Status: appsv1.DeploymentStatus{ - Conditions: []appsv1.DeploymentCondition{ - { - Type: "Available", - Status: "false", + Status: appsv1.DeploymentStatus{ + Conditions: []appsv1.DeploymentCondition{ + { + Type: "Available", + Status: "False", + }, + { + Type: "Progressing", + Status: "True", + }, }, }, }, - }, - ) - }) - Context("with a valid csv", func() { - It("should validate the csv successfully", func() { + ) key := types.NamespacedName{ Name: "test.operator", Namespace: "test-operator-system", } - csv := olmapiv1alpha1.ClusterServiceVersion{ - Spec: olmapiv1alpha1.ClusterServiceVersionSpec{ - DisplayName: "test-operator", - InstallStrategy: olmapiv1alpha1.NamedInstallStrategy{ - StrategySpec: olmapiv1alpha1.StrategyDetailsDeployment{ - DeploymentSpecs: []olmapiv1alpha1.StrategyDeploymentSpec{ - { - Name: "test-operator-controller-manager", - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "control-plane": "controller-manager", - }, - }, + olmclient := Client{KubeClient: fakeClient} + err := olmclient.checkDeploymentErrors(context.TODO(), key, csv) + Expect(err.Error()).To(ContainSubstring("back-off 5m0s restarting failed container")) + }) + + It("check error string for multiple pod failures", func() { + fakeClt := fake.NewFakeClient( + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-jjj", + Namespace: "test-operator-system", + Labels: map[string]string{ + "control-plane": "controller-manager", + }, + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "container1", + Ready: false, + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Message: "Restarting container", }, }, - { - Name: "dummy-operator", - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "dummylabel": "dummyvalue", - }, - }, + }, + }, + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-kkk", + Namespace: "test-operator-system", + Labels: map[string]string{ + "control-plane": "controller-manager", + }, + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "container2", + Ready: false, + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Message: "ImageErrPull", }, }, }, }, }, }, - } - olmclient := Client{KubeClient: fakeClient} - err := olmclient.printDeploymentErrors(context.TODO(), key, csv) - Expect(err).To(BeNil()) - - }) - }) - Context("with csv key namespace NOT provided", func() { - It("should error out ", func() { + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-controller-manager", + Namespace: "test-operator-system", + Labels: map[string]string{ + "control-plane": "controller-manager", + }, + }, + Status: appsv1.DeploymentStatus{ + Conditions: []appsv1.DeploymentCondition{ + { + Type: "Available", + Status: "False", + }, + { + Type: "Progressing", + Status: "True", + }, + }, + }, + }, + ) key := types.NamespacedName{ - Name: "dummy.clusterserviceversion.yaml", + Name: "test.operator", + Namespace: "test-operator-system", } - csv := olmapiv1alpha1.ClusterServiceVersion{ - Spec: olmapiv1alpha1.ClusterServiceVersionSpec{ - DisplayName: "dummy-operator", - InstallStrategy: olmapiv1alpha1.NamedInstallStrategy{ - StrategySpec: olmapiv1alpha1.StrategyDetailsDeployment{ - DeploymentSpecs: []olmapiv1alpha1.StrategyDeploymentSpec{ - { - Name: "dummy-operator", - Spec: appsv1.DeploymentSpec{}, - }, + olmclient := Client{KubeClient: fakeClt} + err := olmclient.checkDeploymentErrors(context.TODO(), key, csv) + Expect(err.Error()).To(ContainSubstring("ImageErrPull")) + Expect(err.Error()).To(ContainSubstring("Restarting container")) + }) + + It("check error string for deployment errors,when no pods exist", func() { + fakeClient = fake.NewFakeClient( + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-controller-manager", + Namespace: "test-operator-system", + Labels: map[string]string{ + "control-plane": "controller-manager", + }, + }, + Status: appsv1.DeploymentStatus{ + Conditions: []appsv1.DeploymentCondition{ + { + Type: "Available", + Status: "False", + Reason: "Pods not available", }, }, }, }, + ) + key := types.NamespacedName{ + Name: "test-operator", + Namespace: "test-operator-system", } olmclient := Client{KubeClient: fakeClient} - err := olmclient.printDeploymentErrors(context.TODO(), key, csv) - Expect(err).ToNot(BeNil()) + err := olmclient.checkDeploymentErrors(context.TODO(), key, csv) + Expect(err.Error()).To(ContainSubstring("Pods not available")) + }) + + It("check error string,when no deployments exist for given CSV", func() { + fakeClient = fake.NewFakeClient() + olmclient := Client{KubeClient: fakeClient} + key := types.NamespacedName{ + Name: "test-operator", + Namespace: "test-operator-system", + } + err := olmclient.checkDeploymentErrors(context.TODO(), key, csv) + Expect(err.Error()).To(ContainSubstring("\"test-operator-controller-manager\" not found")) + Expect(err.Error()).To(ContainSubstring("\"dummy-operator\" not found")) + }) + It("check error string,when no namespace provided", func() { + fakeClient = fake.NewFakeClient() + olmclient := Client{KubeClient: fakeClient} + key := types.NamespacedName{ + Name: "test-operator", + } + err := olmclient.checkDeploymentErrors(context.TODO(), key, csv) + Expect(err.Error()).To(ContainSubstring("no namespace provided to get deployment failures")) }) }) })