Bug 1877408: Add Controller to Normalize the Operator Log Level Values

go.mod

@@ -6,19 +6,19 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
-	github.com/openshift/api v0.0.0-20200730215031-e21882127f24
-	github.com/openshift/build-machinery-go v0.0.0-20200713135615-1f43d26dccc7
-	github.com/openshift/client-go v0.0.0-20200722173614-5a1b0aaeff15
-	github.com/openshift/library-go v0.0.0-20200730143437-a1811581365b
+	github.com/openshift/api v0.0.0-20200827090112-c05698d102cf
+	github.com/openshift/build-machinery-go v0.0.0-20200819073603-48aa266c95f7
+	github.com/openshift/client-go v0.0.0-20200827190008-3062137373b5
+	github.com/openshift/library-go v0.0.0-20200911100307-610c6e9e90b8
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
 	gopkg.in/gcfg.v1 v1.2.0
 	gopkg.in/warnings.v0 v0.1.1 // indirect
-	k8s.io/api v0.19.0-rc.2
-	k8s.io/apimachinery v0.19.0-rc.2
-	k8s.io/client-go v0.19.0-rc.2
-	k8s.io/component-base v0.19.0-rc.2
+	k8s.io/api v0.19.0
+	k8s.io/apimachinery v0.19.0
+	k8s.io/client-go v0.19.0
+	k8s.io/component-base v0.19.0
 	k8s.io/klog v1.0.0
 	sigs.k8s.io/yaml v1.2.0
 )

pkg/operator/operatorloglevel/loglevel_controller.go

@@ -0,0 +1,182 @@
+package operatorloglevel
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	operatorv1 "github.com/openshift/api/operator/v1"
+	"github.com/openshift/library-go/pkg/controller/factory"
+	"github.com/openshift/library-go/pkg/operator/events"
+	"github.com/openshift/library-go/pkg/operator/loglevel"
+	operatorv1helpers "github.com/openshift/library-go/pkg/operator/v1helpers"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/klog"
+)
+
+type operatorLogLevelNormalizer struct {
+	dynamicClient dynamic.Interface
+}
+
+// NewLogLevelNormalizer normalizes the log level in operator custom resource
+// if it's value differs from supported values
+func NewLogLevelNormalizer(dynamicClient dynamic.Interface,
+	operatorClient operatorv1helpers.OperatorClient,
+	recorder events.Recorder) factory.Controller {
+	c := operatorLogLevelNormalizer{
+		dynamicClient: dynamicClient,
+	}
+
+	return factory.New().
+		WithSync(c.sync).
+		WithSyncDegradedOnError(operatorClient).
+		ResyncEvery(5*time.Minute).
+		ToController("OperatorLogLevelNormalizer", recorder)
+}
+
+// sync runs periodically and makes sure that the log level field values in operator
+// spec does not differ from supported values.
+func (c *operatorLogLevelNormalizer) sync(ctx context.Context, syncCtx factory.SyncContext) error {
+	gvrs := []schema.GroupVersionResource{
+		{
+			Group:    "imageregistry.operator.openshift.io",
+			Version:  "v1",
+			Resource: "configs",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "configs",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "etcds",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "kubeapiservers",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "kubecontrollermanagers",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "kubeschedulers",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "openshiftapiservers",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "cloudcredentials",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "kubestorageversionmigrators",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "authentications",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "openshiftcontrollermanagers",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "storages",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "networks",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "consoles",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "csisnapshotcontrollers",
+		},
+		{
+			Group:    "operator.openshift.io",
+			Version:  "v1",
+			Resource: "clustercsidrivers",
+		},
+	}
+
+	for _, gvr := range gvrs {
+		customResources, err := c.dynamicClient.Resource(gvr).List(ctx, metav1.ListOptions{})
+		if err != nil {
+			klog.V(4).Infof("error trying to list custom resources for %s: %v", gvr.Resource, err)
+			continue
+		}
+
+		for _, cr := range customResources.Items {
+			crCopy := cr.DeepCopy()
+			eventMsgs, needsUpdate := normalizeLogLevelField(crCopy, gvr.Resource)
+			if needsUpdate {
+				if _, err := c.dynamicClient.Resource(gvr).Update(ctx, crCopy, metav1.UpdateOptions{}); err != nil {
+					klog.Warningf("failed to normalize log level to %v for operator %s: %v", operatorv1.Normal, gvr.Resource, err)
+					continue
+				}
+				for _, event := range eventMsgs {
+					syncCtx.Recorder().Event("OperatorLogLevelChange", event)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func normalizeLogLevelField(cr *unstructured.Unstructured,
+	resourceName string) ([]string, bool) {
+	needsUpdate := false
+	eventMsgs := []string{}
+
+	for _, logLevelFieldPath := range [][]string{{"spec", "operatorLogLevel"}, {"spec", "logLevel"}} {
+		// custom resources that do not have log level field are ignored.
+		currentLogLevel, ok, err := unstructured.NestedString(cr.UnstructuredContent(), logLevelFieldPath...)
+		if err != nil {
+			klog.V(4).Infof("failed to find %q in custom resource %s: %v", strings.Join(logLevelFieldPath, "."), resourceName, err)
+			continue
+		}
+		if !ok {
+			continue
+		}
+
+		if loglevel.ValidLogLevel(operatorv1.LogLevel(currentLogLevel)) {
+			continue
+		}
+
+		if err := unstructured.SetNestedField(cr.UnstructuredContent(), string(operatorv1.Normal), logLevelFieldPath...); err != nil {
+			klog.Warningf("failed to set log level to %s in resource %s", operatorv1.Normal, resourceName)
+			continue
+		}
+
+		eventMsgs = append(eventMsgs, fmt.Sprintf("%q changed from %q to %q", strings.Join(logLevelFieldPath, "."), currentLogLevel, operatorv1.Normal))
+		needsUpdate = true
+	}
+
+	return eventMsgs, needsUpdate
+}

pkg/operator/operatorloglevel/loglevel_controller_test.go

@@ -0,0 +1,130 @@
+package operatorloglevel
+
+import (
+	"reflect"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+func Test_normalizeLogLevelField(t *testing.T) {
+	type args struct {
+		cr           *unstructured.Unstructured
+		resourceName string
+	}
+	tests := []struct {
+		name           string
+		args           args
+		expectedEvents []string
+		expectUpdate   bool
+	}{
+		{
+			name: "log levels not set",
+			args: args{
+				cr: &unstructured.Unstructured{},
+			},
+			expectedEvents: []string{},
+		},
+		{
+			name: "invalid operator log level",
+			args: args{
+				cr: &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"spec": map[string]interface{}{
+							"operatorLogLevel": "test",
+						},
+					},
+				},
+				resourceName: "dummy",
+			},
+			expectedEvents: []string{"\"spec.operatorLogLevel\" changed from \"test\" to \"Normal\""},
+			expectUpdate:   true,
+		},
+		{
+			name: "invalid operand log level",
+			args: args{
+				cr: &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"spec": map[string]interface{}{
+							"logLevel": "test",
+						},
+					},
+				},
+				resourceName: "dummy",
+			},
+			expectedEvents: []string{"\"spec.logLevel\" changed from \"test\" to \"Normal\""},
+			expectUpdate:   true,
+		},
+		{
+			name: "invalid operator and operand log level",
+			args: args{
+				cr: &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"spec": map[string]interface{}{
+							"logLevel":         "test",
+							"operatorLogLevel": "test",
+						},
+					},
+				},
+				resourceName: "dummy",
+			},
+			expectedEvents: []string{"\"spec.operatorLogLevel\" changed from \"test\" to \"Normal\"",
+				"\"spec.logLevel\" changed from \"test\" to \"Normal\""},
+			expectUpdate: true,
+		},
+		{
+			name: "valid operator and operand log level",
+			args: args{
+				cr: &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"spec": map[string]interface{}{
+							"logLevel":         "Trace",
+							"operatorLogLevel": "TraceAll",
+						},
+					},
+				},
+				resourceName: "dummy",
+			},
+			expectedEvents: []string{},
+		},
+		{
+			name: "valid operator log level",
+			args: args{
+				cr: &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"spec": map[string]interface{}{
+							"operatorLogLevel": "Normal",
+						},
+					},
+				},
+				resourceName: "dummy",
+			},
+			expectedEvents: []string{},
+		},
+		{
+			name: "valid operand log level",
+			args: args{
+				cr: &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"spec": map[string]interface{}{
+							"logLevel": "Debug",
+						},
+					},
+				},
+				resourceName: "dummy",
+			},
+			expectedEvents: []string{},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			events, needsUpdate := normalizeLogLevelField(tt.args.cr, tt.args.resourceName)
+			if !reflect.DeepEqual(events, tt.expectedEvents) {
+				t.Errorf("normalizeLogLevelField() events = %v, want %v", events, tt.expectedEvents)
+			}
+			if needsUpdate != tt.expectUpdate {
+				t.Errorf("normalizeLogLevelField() needsUpdate = %v, want %v", needsUpdate, tt.expectUpdate)
+			}
+		})
+	}
+}

pkg/operator/starter.go

@@ -8,6 +8,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 
 	configv1 "github.com/openshift/api/config/v1"
@@ -25,6 +26,7 @@ import (
 	"github.com/openshift/cluster-config-operator/pkg/operator/kube_cloud_config"
 	"github.com/openshift/cluster-config-operator/pkg/operator/migration_aws_status"
 	"github.com/openshift/cluster-config-operator/pkg/operator/operatorclient"
+	"github.com/openshift/cluster-config-operator/pkg/operator/operatorloglevel"
 )
 
 func RunOperator(ctx context.Context, controllerContext *controllercmd.ControllerContext) error {
@@ -37,6 +39,11 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 	if err != nil {
 		return err
 	}
+	dynamicClient, err := dynamic.NewForConfig(controllerContext.KubeConfig)
+	if err != nil {
+		return nil
+	}
+
 	configInformers := configv1informers.NewSharedInformerFactory(configClient, 10*time.Minute)
 	kubeInformersForNamespaces := v1helpers.NewKubeInformersForNamespaces(kubeClient,
 		"",
@@ -57,6 +64,8 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 		controllerContext.EventRecorder,
 	)
 
+	logLevelNormalizer := operatorloglevel.NewLogLevelNormalizer(dynamicClient, operatorClient, controllerContext.EventRecorder)
+
 	kubeCloudConfigController := kubecloudconfig.NewController(
 		operatorClient,
 		configClient.ConfigV1(),
@@ -141,6 +150,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 	go statusController.Run(ctx, 1)
 	go operatorController.Run(ctx, 1)
 	go migrationAWSStatusController.Run(ctx, 1)
+	go logLevelNormalizer.Run(ctx, 1)
 
 	<-ctx.Done()
 	return nil