From 5527f842eda1fd51538590a592d3c95a3cd679d9 Mon Sep 17 00:00:00 2001 From: Harshal Patil Date: Fri, 9 Apr 2021 11:56:56 +0530 Subject: [PATCH] Wait for controller config before applying kubeletconfig Signed-off-by: Ryan Phillips --- .../kubelet_config_controller.go | 17 ++++++++++++++-- .../kubelet_config_controller_test.go | 20 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 3da3071689..be223355e6 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -49,8 +49,8 @@ const ( // With the current rate-limiter in use (5ms*2^(maxRetries-1)) the following numbers represent the times // a machineconfig pool is going to be requeued: // - // 5ms, 10ms, 20ms, 40ms, 80ms, 160ms, 320ms, 640ms, 1.3s, 2.6s, 5.1s, 10.2s, 20.4s, 41s, 82s - maxRetries = 15 + // 18 allows for retries up to about 10 minutes to allow for slower machines to catchup. + maxRetries = 18 ) var ( @@ -394,6 +394,11 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { glog.V(4).Infof("Finished syncing kubeletconfig %q (%v)", key, time.Since(startTime)) }() + // Wait to apply a kubelet config if the controller config is not completed + if err := mcfgv1.IsControllerConfigCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + return err + } + _, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { return err @@ -458,6 +463,14 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { } for _, pool := range mcpPools { + if pool.Spec.Configuration.Name == "" { + updateDelay := 5 * time.Second + // Previously we spammed the logs about empty pools. + // Let's just pause for a bit here to let the renderer + // initialize them. + time.Sleep(updateDelay) + return fmt.Errorf("Pool %s is unconfigured, pausing %v for renderer to initialize", pool.Name, updateDelay) + } role := pool.Name // Get MachineConfig managedKey, err := getManagedKubeletConfigKey(pool, ctrl.client, cfg) diff --git a/pkg/controller/kubelet-config/kubelet_config_controller_test.go b/pkg/controller/kubelet-config/kubelet_config_controller_test.go index f5c0c9601e..2bb293789b 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller_test.go @@ -11,6 +11,7 @@ import ( osev1 "github.com/openshift/api/config/v1" oseconfigfake "github.com/openshift/client-go/config/clientset/versioned/fake" oseinformersv1 "github.com/openshift/client-go/config/informers/externalversions" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -111,6 +112,25 @@ func newControllerConfig(name string, platform osev1.PlatformType) *mcfgv1.Contr }, }, }, + Status: mcfgv1.ControllerConfigStatus{ + Conditions: []mcfgv1.ControllerConfigStatusCondition{ + { + Type: mcfgv1.TemplateControllerCompleted, + Status: corev1.ConditionTrue, + Message: "", + }, + { + Type: mcfgv1.TemplateControllerRunning, + Status: corev1.ConditionFalse, + Message: "", + }, + { + Type: mcfgv1.TemplateControllerFailing, + Status: corev1.ConditionFalse, + Message: "", + }, + }, + }, } return cc }