From 1fdf44578d0f9ac1c6be227f919b40829b488905 Mon Sep 17 00:00:00 2001 From: bharathi-tenneti Date: Mon, 31 Aug 2020 11:31:51 -0400 Subject: [PATCH 1/3] Added deployment status check --- internal/olm/client/client.go | 71 +++++++- internal/olm/client/client_suite_test.go | 27 ++++ internal/olm/client/client_test.go | 196 +++++++++++++++++++++++ 3 files changed, 292 insertions(+), 2 deletions(-) create mode 100644 internal/olm/client/client_suite_test.go create mode 100644 internal/olm/client/client_test.go diff --git a/internal/olm/client/client.go b/internal/olm/client/client.go index 5edfb4f873..da5ca77165 100644 --- a/internal/olm/client/client.go +++ b/internal/olm/client/client.go @@ -29,6 +29,8 @@ import ( olmapiv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" log "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -194,8 +196,8 @@ func (c Client) DoCSVWait(ctx context.Context, key types.NamespacedName) error { ) once := sync.Once{} + csv := olmapiv1alpha1.ClusterServiceVersion{} csvPhaseSucceeded := func() (bool, error) { - csv := olmapiv1alpha1.ClusterServiceVersion{} err := c.KubeClient.Get(ctx, key, &csv) if err != nil { if apierrors.IsNotFound(err) { @@ -222,7 +224,72 @@ func (c Client) DoCSVWait(ctx context.Context, key types.NamespacedName) error { } } - return wait.PollImmediateUntil(time.Second, csvPhaseSucceeded, ctx.Done()) + err := wait.PollImmediateUntil(time.Second, csvPhaseSucceeded, ctx.Done()) + if err != nil && errors.Is(err, context.DeadlineExceeded) { + if depCheckErr := c.printDeploymentErrors(ctx, key, csv); depCheckErr != nil { + return fmt.Errorf("failed to run operator: %v %v", err, depCheckErr) + } + } + return err +} + +// printDeploymentErrors function loops through deployment specs of a given CSV, and prints reason +// in case of failures, based on deployment condition. +func (c Client) printDeploymentErrors(ctx context.Context, key types.NamespacedName, csv olmapiv1alpha1.ClusterServiceVersion) error { + if key.Namespace == "" { + return fmt.Errorf("no namespace provided to get deployment failures") + } + dep := &appsv1.Deployment{} + for _, ds := range csv.Spec.InstallStrategy.StrategySpec.DeploymentSpecs { + depKey := types.NamespacedName{ + Namespace: key.Namespace, + Name: ds.Name, + } + depSelectors := ds.Spec.Selector + if err := c.KubeClient.Get(ctx, depKey, dep); err != nil { + log.Printf("failed to run operator, deployment not found for : %v\n", ds.Name) + continue + } + for _, s := range dep.Status.Conditions { + if s.Type == appsv1.DeploymentAvailable && s.Status == corev1.ConditionFalse { + log.Printf("failed to run operator: deployment failed for : %v\n, with reason %v\n", ds.Name, s.Reason) + if err := c.printPodErrors(ctx, depSelectors); err != nil { + return err + } + } + } + } + return nil +} + +// printPodErrors loops through pods, and prints pod errors if any. +func (c Client) printPodErrors(ctx context.Context, depSelectors *metav1.LabelSelector) error { + // loop through pods and return specific error message. + podErrors := make(map[string]string) + podList := &corev1.PodList{} + podLabelSelectors, err := metav1.LabelSelectorAsSelector(depSelectors) + if err != nil { + return err + } + options := client.ListOptions{ + LabelSelector: podLabelSelectors, + } + if err := c.KubeClient.List(ctx, podList, &options); err != nil { + return fmt.Errorf("error getting Pods: %v", err) + } + for _, p := range podList.Items { + if p.Status.Phase != corev1.PodSucceeded { + for _, cs := range p.Status.ContainerStatuses { + if !cs.Ready { + podErrors[p.Name] = cs.State.Waiting.Message + } + } + } + } + if len(podErrors) > 0 { + log.Printf("pod errors: %v\n", podErrors) + } + return nil } // GetInstalledVersion returns the OLM version installed in the namespace informed. diff --git a/internal/olm/client/client_suite_test.go b/internal/olm/client/client_suite_test.go new file mode 100644 index 0000000000..facb66e344 --- /dev/null +++ b/internal/olm/client/client_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "check depErrors") +} diff --git a/internal/olm/client/client_test.go b/internal/olm/client/client_test.go new file mode 100644 index 0000000000..42f1619e95 --- /dev/null +++ b/internal/olm/client/client_test.go @@ -0,0 +1,196 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + olmapiv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/types" + + "sigs.k8s.io/controller-runtime/pkg/client" + fake "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var _ = Describe("Client", func() { + Describe("printDeploymentErrors", func() { + + var ( + fakeClient client.Client + ) + + BeforeEach(func() { + fakeClient = fake.NewFakeClient( + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-controller-manager-8687c65f7d-kc44t", + Namespace: "test-operator-system", + Labels: map[string]string{ + "control-plane": "controller-manager", + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + { + Ready: false, + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Message: "back-off 5m0s restarting failed container)", + Reason: "CrashLoopBackOff", + }, + }, + }, + }, + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummypod", + Namespace: "testns", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-controller-manager-jjj", + Namespace: "test-operator-system", + Labels: map[string]string{ + "control-plane": "controller-manager", + }, + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Ready: true, + }, + }, + }, + }, + + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator-controller-manager", + Namespace: "test-operator-system", + Labels: map[string]string{ + "control-plane": "controller-manager", + }, + }, + Status: appsv1.DeploymentStatus{ + Conditions: []appsv1.DeploymentCondition{ + { + Type: "Available", + Status: "False", + Reason: "MinimumReplicasUnavailable", + }, + { + Type: "Progressing", + Status: "True", + Reason: "NewReplicaSetAvailable", + }, + }, + }, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-operator", + Namespace: "dummy-operator-system", + }, + Status: appsv1.DeploymentStatus{ + Conditions: []appsv1.DeploymentCondition{ + { + Type: "Available", + Status: "false", + }, + }, + }, + }, + ) + }) + Context("with a valid csv", func() { + It("should validate the csv successfully", func() { + key := types.NamespacedName{ + Name: "test.clusterserviceversion.yaml", + Namespace: "test-operator-system", + } + csv := olmapiv1alpha1.ClusterServiceVersion{ + Spec: olmapiv1alpha1.ClusterServiceVersionSpec{ + DisplayName: "test-operator", + InstallStrategy: olmapiv1alpha1.NamedInstallStrategy{ + StrategySpec: olmapiv1alpha1.StrategyDetailsDeployment{ + DeploymentSpecs: []olmapiv1alpha1.StrategyDeploymentSpec{ + { + Name: "test-operator-controller-manager", + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "control-plane": "controller-manager", + }, + }, + }, + }, + { + Name: "dummy-operator", + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "dummylabel": "dummyvalue", + }, + }, + }, + }, + }, + }, + }, + }, + } + olmclient := Client{KubeClient: fakeClient} + err := olmclient.printDeploymentErrors(context.TODO(), key, csv) + Expect(err).To(BeNil()) + + }) + }) + Context("with csv key namespace NOT provided", func() { + It("should error out ", func() { + key := types.NamespacedName{ + Name: "dummy.clusterserviceversion.yaml", + } + csv := olmapiv1alpha1.ClusterServiceVersion{ + Spec: olmapiv1alpha1.ClusterServiceVersionSpec{ + DisplayName: "dummy-operator", + InstallStrategy: olmapiv1alpha1.NamedInstallStrategy{ + StrategySpec: olmapiv1alpha1.StrategyDetailsDeployment{ + DeploymentSpecs: []olmapiv1alpha1.StrategyDeploymentSpec{ + { + Name: "dummy-operator", + Spec: appsv1.DeploymentSpec{}, + }, + }, + }, + }, + }, + } + olmclient := Client{KubeClient: fakeClient} + err := olmclient.printDeploymentErrors(context.TODO(), key, csv) + Expect(err).ToNot(BeNil()) + }) + }) + }) +}) From 63fc9fb1876c2a6438e6a5159a7e42de1bf8e68c Mon Sep 17 00:00:00 2001 From: bharathi-tenneti Date: Wed, 9 Sep 2020 13:49:39 -0400 Subject: [PATCH 2/3] Add namespace filter for podlist --- internal/olm/client/client.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/olm/client/client.go b/internal/olm/client/client.go index da5ca77165..524d0e1423 100644 --- a/internal/olm/client/client.go +++ b/internal/olm/client/client.go @@ -253,7 +253,7 @@ func (c Client) printDeploymentErrors(ctx context.Context, key types.NamespacedN for _, s := range dep.Status.Conditions { if s.Type == appsv1.DeploymentAvailable && s.Status == corev1.ConditionFalse { log.Printf("failed to run operator: deployment failed for : %v\n, with reason %v\n", ds.Name, s.Reason) - if err := c.printPodErrors(ctx, depSelectors); err != nil { + if err := c.printPodErrors(ctx, depSelectors, key); err != nil { return err } } @@ -263,7 +263,7 @@ func (c Client) printDeploymentErrors(ctx context.Context, key types.NamespacedN } // printPodErrors loops through pods, and prints pod errors if any. -func (c Client) printPodErrors(ctx context.Context, depSelectors *metav1.LabelSelector) error { +func (c Client) printPodErrors(ctx context.Context, depSelectors *metav1.LabelSelector, key types.NamespacedName) error { // loop through pods and return specific error message. podErrors := make(map[string]string) podList := &corev1.PodList{} @@ -273,6 +273,7 @@ func (c Client) printPodErrors(ctx context.Context, depSelectors *metav1.LabelSe } options := client.ListOptions{ LabelSelector: podLabelSelectors, + Namespace: key.Namespace, } if err := c.KubeClient.List(ctx, podList, &options); err != nil { return fmt.Errorf("error getting Pods: %v", err) From 6b8b0623a514274ac536cf1ca672db681ccbc4c2 Mon Sep 17 00:00:00 2001 From: bharathi-tenneti Date: Wed, 9 Sep 2020 15:43:32 -0400 Subject: [PATCH 3/3] Added TODO for refactor --- internal/olm/client/client.go | 7 ++++--- internal/olm/client/client_test.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/olm/client/client.go b/internal/olm/client/client.go index 524d0e1423..51ba991f06 100644 --- a/internal/olm/client/client.go +++ b/internal/olm/client/client.go @@ -227,12 +227,13 @@ func (c Client) DoCSVWait(ctx context.Context, key types.NamespacedName) error { err := wait.PollImmediateUntil(time.Second, csvPhaseSucceeded, ctx.Done()) if err != nil && errors.Is(err, context.DeadlineExceeded) { if depCheckErr := c.printDeploymentErrors(ctx, key, csv); depCheckErr != nil { - return fmt.Errorf("failed to run operator: %v %v", err, depCheckErr) + return fmt.Errorf("error printing operator resource errors: %v %v", err, depCheckErr) } } return err } +// TODO(btenneti) Refactor function to collect errors into customized error and return. // printDeploymentErrors function loops through deployment specs of a given CSV, and prints reason // in case of failures, based on deployment condition. func (c Client) printDeploymentErrors(ctx context.Context, key types.NamespacedName, csv olmapiv1alpha1.ClusterServiceVersion) error { @@ -247,12 +248,12 @@ func (c Client) printDeploymentErrors(ctx context.Context, key types.NamespacedN } depSelectors := ds.Spec.Selector if err := c.KubeClient.Get(ctx, depKey, dep); err != nil { - log.Printf("failed to run operator, deployment not found for : %v\n", ds.Name) + log.Printf("error getting operator deployment %q: %v", ds.Name, err) continue } for _, s := range dep.Status.Conditions { if s.Type == appsv1.DeploymentAvailable && s.Status == corev1.ConditionFalse { - log.Printf("failed to run operator: deployment failed for : %v\n, with reason %v\n", ds.Name, s.Reason) + log.Printf("operator deployment %q not available: %s", ds.Name, s.Reason) if err := c.printPodErrors(ctx, depSelectors, key); err != nil { return err } diff --git a/internal/olm/client/client_test.go b/internal/olm/client/client_test.go index 42f1619e95..30225398b0 100644 --- a/internal/olm/client/client_test.go +++ b/internal/olm/client/client_test.go @@ -127,7 +127,7 @@ var _ = Describe("Client", func() { Context("with a valid csv", func() { It("should validate the csv successfully", func() { key := types.NamespacedName{ - Name: "test.clusterserviceversion.yaml", + Name: "test.operator", Namespace: "test-operator-system", } csv := olmapiv1alpha1.ClusterServiceVersion{