From 719610ed27ad01497617510dfab88a89e217f8ae Mon Sep 17 00:00:00 2001 From: "Frank A. Zdarsky" Date: Fri, 10 Sep 2021 14:47:56 +0200 Subject: [PATCH 1/2] Fix panic on buggy kustomization, add retries Signed-off-by: Frank A. Zdarsky --- pkg/kustomize/apply.go | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/pkg/kustomize/apply.go b/pkg/kustomize/apply.go index 9863236608..493d15356d 100644 --- a/pkg/kustomize/apply.go +++ b/pkg/kustomize/apply.go @@ -6,6 +6,7 @@ import ( "flag" "os" "path/filepath" + "time" "github.com/openshift/microshift/pkg/config" "github.com/openshift/microshift/pkg/util" @@ -15,10 +16,14 @@ import ( cliflag "k8s.io/component-base/cli/flag" "k8s.io/kubectl/pkg/cmd/apply" cmdutil "k8s.io/kubectl/pkg/cmd/util" - "k8s.io/kubectl/pkg/util/i18n" "k8s.io/kubectl/pkg/util/templates" ) +const ( + retries = 3 + retryInterval = 10 * time.Second +) + type Kustomizer struct { path string kubeconfig string @@ -41,7 +46,11 @@ func (s *Kustomizer) Run(ctx context.Context, ready chan<- struct{}, stopped cha kustomization := filepath.Join(s.path, "kustomization.yaml") if _, err := os.Stat(kustomization); !errors.Is(err, os.ErrNotExist) { logrus.Infof("Applying kustomization at " + kustomization) - ApplyKustomization(s.path, s.kubeconfig) + if err := ApplyKustomizationWithRetries(s.path, s.kubeconfig); err != nil { + logrus.Warnf("Applying kustomization failed: %s. Giving up.", err) + } else { + logrus.Infof("Kustomization applied successfully.") + } } else { logrus.Infof("No kustomization found at " + kustomization) } @@ -49,13 +58,23 @@ func (s *Kustomizer) Run(ctx context.Context, ready chan<- struct{}, stopped cha return ctx.Err() } -func ApplyKustomization(path string, kubeconfig string) error { +func ApplyKustomizationWithRetries(kustomization string, kubeconfig string) error { + err := ApplyKustomization(kustomization, kubeconfig) + for retry := 1; retry <= retries && err != nil; retry++ { + logrus.Infof("Applying kustomization failed: %s. Retrying in %s.", err, retryInterval) + time.Sleep(retryInterval) + err = ApplyKustomization(kustomization, kubeconfig) + } + return err +} + +func ApplyKustomization(kustomization string, kubeconfig string) error { cmds := &cobra.Command{ Use: "kubectl", - Short: i18n.T("kubectl controls the Kubernetes cluster manager"), + Short: "kubectl", } flags := cmds.PersistentFlags() - flags.SetNormalizeFunc(cliflag.WarnWordSepNormalizeFunc) // Warn for "_" flags + flags.SetNormalizeFunc(cliflag.WarnWordSepNormalizeFunc) flags.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag() @@ -79,11 +98,14 @@ func ApplyKustomization(path string, kubeconfig string) error { args := []string{ "--kubeconfig=" + kubeconfig, - "-k", path, + "-k", kustomization, } - util.Must(applyCommand.ParseFlags(args)) - applyCommand.Run(applyCommand, nil) - return nil + o := apply.NewApplyOptions(ioStreams) + o.DeleteFlags.FileNameFlags.Kustomize = &kustomization + if err := o.Complete(f, applyCommand); err != nil { + return err + } + return o.Run() } From 252ba968d1fb38f34964ea93883dc187d38d4f88 Mon Sep 17 00:00:00 2001 From: "Frank A. Zdarsky" Date: Fri, 10 Sep 2021 18:50:00 +0200 Subject: [PATCH 2/2] Use wait.Poll() Signed-off-by: Frank A. Zdarsky --- pkg/kustomize/apply.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/kustomize/apply.go b/pkg/kustomize/apply.go index 493d15356d..3ba94332b4 100644 --- a/pkg/kustomize/apply.go +++ b/pkg/kustomize/apply.go @@ -12,6 +12,7 @@ import ( "github.com/openshift/microshift/pkg/util" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cli-runtime/pkg/genericclioptions" cliflag "k8s.io/component-base/cli/flag" "k8s.io/kubectl/pkg/cmd/apply" @@ -20,8 +21,8 @@ import ( ) const ( - retries = 3 retryInterval = 10 * time.Second + retryTimeout = 1 * time.Minute ) type Kustomizer struct { @@ -59,13 +60,13 @@ func (s *Kustomizer) Run(ctx context.Context, ready chan<- struct{}, stopped cha } func ApplyKustomizationWithRetries(kustomization string, kubeconfig string) error { - err := ApplyKustomization(kustomization, kubeconfig) - for retry := 1; retry <= retries && err != nil; retry++ { - logrus.Infof("Applying kustomization failed: %s. Retrying in %s.", err, retryInterval) - time.Sleep(retryInterval) - err = ApplyKustomization(kustomization, kubeconfig) - } - return err + return wait.Poll(retryInterval, retryTimeout, func() (bool, error) { + if err := ApplyKustomization(kustomization, kubeconfig); err != nil { + logrus.Infof("Applying kustomization failed: %s. Retrying in %s.", err, retryInterval) + return false, nil + } + return true, nil + }) } func ApplyKustomization(kustomization string, kubeconfig string) error {