From fda33fd134f54a9d60f1d3198a65f074243fd64a Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 27 Apr 2020 14:45:36 +0200 Subject: [PATCH 01/15] Add Leader election flags for nodelink --- cmd/nodelink-controller/main.go | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/cmd/nodelink-controller/main.go b/cmd/nodelink-controller/main.go index ca0d0242f7..73bae14dac 100644 --- a/cmd/nodelink-controller/main.go +++ b/cmd/nodelink-controller/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "runtime" + "time" mapiv1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1" "github.com/openshift/machine-api-operator/pkg/controller" @@ -23,7 +24,30 @@ func printVersion() { func main() { printVersion() - watchNamespace := flag.String("namespace", "", "Namespace that the controller watches to reconcile machine-api objects. If unspecified, the controller watches for machine-api objects across all namespaces.") + watchNamespace := flag.String( + "namespace", + "", + "Namespace that the controller watches to reconcile machine-api objects. If unspecified, the controller watches for machine-api objects across all namespaces.", + ) + + leaderElectResourceNamespace := flag.String( + "leader-elect-resource-namespace", + "", + "The namespace of resource object that is used for locking during leader election. If unspecified and running in cluster, defaults to the service account namespace for the controller. Required for leader-election outside of a cluster.", + ) + + leaderElect := flag.Bool( + "leader-elect", + false, + "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.", + ) + + leaderElectLeaseDuration := flag.Duration( + "leader-elect-lease-duration", + 15*time.Second, + "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", + ) + klog.InitFlags(nil) flag.Set("logtostderr", "true") flag.Parse() @@ -36,7 +60,11 @@ func main() { opts := manager.Options{ // Disable metrics serving - MetricsBindAddress: "0", + MetricsBindAddress: "0", + LeaderElection: *leaderElect, + LeaderElectionNamespace: *leaderElectResourceNamespace, + LeaderElectionID: "cluster-api-provider-nodelink-leader", + LeaseDuration: leaderElectLeaseDuration, } if *watchNamespace != "" { opts.Namespace = *watchNamespace From a72ad436183316c853a3f670d73babb30223173b Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Thu, 2 Jul 2020 13:24:35 +0200 Subject: [PATCH 02/15] Add Leader election flags for vsphere --- cmd/vsphere/main.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/cmd/vsphere/main.go b/cmd/vsphere/main.go index 4e916a750f..af2d2e48d8 100644 --- a/cmd/vsphere/main.go +++ b/cmd/vsphere/main.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "time" configv1 "github.com/openshift/api/config/v1" "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1" @@ -22,7 +23,30 @@ func main() { flag.BoolVar(&printVersion, "version", false, "print version and exit") klog.InitFlags(nil) - watchNamespace := flag.String("namespace", "", "Namespace that the controller watches to reconcile machine-api objects. If unspecified, the controller watches for machine-api objects across all namespaces.") + watchNamespace := flag.String( + "namespace", + "", + "Namespace that the controller watches to reconcile machine-api objects. If unspecified, the controller watches for machine-api objects across all namespaces.", + ) + + leaderElectResourceNamespace := flag.String( + "leader-elect-resource-namespace", + "", + "The namespace of resource object that is used for locking during leader election. If unspecified and running in cluster, defaults to the service account namespace for the controller. Required for leader-election outside of a cluster.", + ) + + leaderElect := flag.Bool( + "leader-elect", + false, + "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.", + ) + + leaderElectLeaseDuration := flag.Duration( + "leader-elect-lease-duration", + 15*time.Second, + "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", + ) + flag.Set("logtostderr", "true") flag.Parse() @@ -35,8 +59,13 @@ func main() { opts := manager.Options{ // Disable metrics serving - MetricsBindAddress: "0", + MetricsBindAddress: "0", + LeaderElection: *leaderElect, + LeaderElectionNamespace: *leaderElectResourceNamespace, + LeaderElectionID: "cluster-api-provider-vsphere-leader", + LeaseDuration: leaderElectLeaseDuration, } + if *watchNamespace != "" { opts.Namespace = *watchNamespace klog.Infof("Watching machine-api objects only in namespace %q for reconciliation.", opts.Namespace) From d6ec8b6c6be5a1fd0fb008518dc5d1dc3a26187e Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Thu, 2 Jul 2020 13:29:19 +0200 Subject: [PATCH 03/15] Add Leader election flags for machine health checks --- cmd/machine-healthcheck/main.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/cmd/machine-healthcheck/main.go b/cmd/machine-healthcheck/main.go index 4412924ec6..5ad1c171d7 100644 --- a/cmd/machine-healthcheck/main.go +++ b/cmd/machine-healthcheck/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "runtime" + "time" "github.com/openshift/machine-api-operator/pkg/controller/machinehealthcheck" @@ -23,7 +24,30 @@ func printVersion() { } func main() { - watchNamespace := flag.String("namespace", "", "Namespace that the controller watches to reconcile machine-api objects. If unspecified, the controller watches for machine-api objects across all namespaces.") + watchNamespace := flag.String( + "namespace", + "", + "Namespace that the controller watches to reconcile machine-api objects. If unspecified, the controller watches for machine-api objects across all namespaces.", + ) + + leaderElectResourceNamespace := flag.String( + "leader-elect-resource-namespace", + "", + "The namespace of resource object that is used for locking during leader election. If unspecified and running in cluster, defaults to the service account namespace for the controller. Required for leader-election outside of a cluster.", + ) + + leaderElect := flag.Bool( + "leader-elect", + false, + "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.", + ) + + leaderElectLeaseDuration := flag.Duration( + "leader-elect-lease-duration", + 15*time.Second, + "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", + ) + flag.Parse() printVersion() @@ -35,8 +59,13 @@ func main() { opts := manager.Options{ // Disable metrics serving - MetricsBindAddress: "0", + MetricsBindAddress: "0", + LeaderElection: *leaderElect, + LeaderElectionNamespace: *leaderElectResourceNamespace, + LeaderElectionID: "cluster-api-provider-healthcheck-leader", + LeaseDuration: leaderElectLeaseDuration, } + if *watchNamespace != "" { opts.Namespace = *watchNamespace glog.Infof("Watching machine-api objects only in namespace %q for reconciliation.", opts.Namespace) From 644ed9aee2f6f37e39b0b5f759fc2f23d5b93f8f Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Thu, 2 Jul 2020 13:32:40 +0200 Subject: [PATCH 04/15] Add Leader election flags for machineSet controller --- cmd/machineset/main.go | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/cmd/machineset/main.go b/cmd/machineset/main.go index 590e4a4ac4..ade6e6543b 100644 --- a/cmd/machineset/main.go +++ b/cmd/machineset/main.go @@ -37,6 +37,24 @@ func main() { watchNamespace := flag.String("namespace", "", "Namespace that the controller watches to reconcile cluster-api objects. If unspecified, the controller watches for cluster-api objects across all namespaces.") + leaderElectResourceNamespace := flag.String( + "leader-elect-resource-namespace", + "", + "The namespace of resource object that is used for locking during leader election. If unspecified and running in cluster, defaults to the service account namespace for the controller. Required for leader-election outside of a cluster.", + ) + + leaderElect := flag.Bool( + "leader-elect", + false, + "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.", + ) + + leaderElectLeaseDuration := flag.Duration( + "leader-elect-lease-duration", + 15*time.Second, + "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", + ) + flag.Parse() if *watchNamespace != "" { log.Printf("Watching cluster-api objects only in namespace %q for reconciliation.", *watchNamespace) @@ -52,9 +70,13 @@ func main() { syncPeriod := 10 * time.Minute opts := manager.Options{ // Disable metrics serving - MetricsBindAddress: "0", - SyncPeriod: &syncPeriod, - Namespace: *watchNamespace, + MetricsBindAddress: "0", + SyncPeriod: &syncPeriod, + Namespace: *watchNamespace, + LeaderElection: *leaderElect, + LeaderElectionNamespace: *leaderElectResourceNamespace, + LeaderElectionID: "cluster-api-provider-machineset-leader", + LeaseDuration: leaderElectLeaseDuration, } mgr, err := manager.New(cfg, opts) if err != nil { From 303f931e9b4f1862a323aa3f06cc56731e6ebbfb Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Fri, 3 Jul 2020 10:54:27 +0200 Subject: [PATCH 05/15] Set leader-election for all controllers enabled Using leader election by default will add stronger guarantees than we have today that only one controller is running at a time to protect against edge cases where the deployment replica could be increased or upgrades with permissive maxSurge. Relevant provider PRs: - https://github.com/openshift/cluster-api-provider-gcp/pull/85 - https://github.com/openshift/cluster-api-provider-aws/pull/315 - https://github.com/openshift/cluster-api-provider-azure/pull/122 - https://github.com/openshift/cluster-api-provider-openstack/pull/108 - https://github.com/openshift/cluster-api-provider-baremetal/pull/81 - https://github.com/openshift/cluster-api-provider-ovirt/pull/55 - https://github.com/openshift/machine-api-operator/pull/571 --- pkg/operator/sync.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go index 00b3058951..4abbc1acac 100644 --- a/pkg/operator/sync.go +++ b/pkg/operator/sync.go @@ -313,6 +313,7 @@ func newContainers(config *OperatorConfig, features map[string]bool) []corev1.Co args := []string{ "--logtostderr=true", "--v=3", + "--leader-elect=true", fmt.Sprintf("--namespace=%s", config.TargetNamespace), } From 91251df95beb0f0c7e17e483bb6200bfedbb5dfe Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 20 Jul 2020 12:56:21 +0200 Subject: [PATCH 06/15] Increate leader election lease time for vsphere The machine-api-controller components are refreshing their lease more than all other components combined. Bringing this to 90s each, will decrease etcd writes at idle. --- cmd/vsphere/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/vsphere/main.go b/cmd/vsphere/main.go index af2d2e48d8..2a8fa8e01e 100644 --- a/cmd/vsphere/main.go +++ b/cmd/vsphere/main.go @@ -43,7 +43,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 15*time.Second, + 90*time.Second, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) From a52a7ec808946148f36be57675f51bbfbc8a0b3a Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 20 Jul 2020 12:57:14 +0200 Subject: [PATCH 07/15] Increate leader election lease time for nodelink The machine-api-controller components are refreshing their lease more than all other components combined. Bringing this to 90s each, will decrease etcd writes at idle. --- cmd/nodelink-controller/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/nodelink-controller/main.go b/cmd/nodelink-controller/main.go index 73bae14dac..99572c5a54 100644 --- a/cmd/nodelink-controller/main.go +++ b/cmd/nodelink-controller/main.go @@ -44,7 +44,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 15*time.Second, + 90*time.Second, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) From 3f124196d3800473b55d9c4a7cac8b02170062d0 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 20 Jul 2020 12:57:45 +0200 Subject: [PATCH 08/15] Increate leader election lease time for machineset The machine-api-controller components are refreshing their lease more than all other components combined. Bringing this to 90s each, will decrease etcd writes at idle. --- cmd/machineset/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/machineset/main.go b/cmd/machineset/main.go index ade6e6543b..61bf0ecfbd 100644 --- a/cmd/machineset/main.go +++ b/cmd/machineset/main.go @@ -51,7 +51,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 15*time.Second, + 90*time.Second, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) From 0a561840f8f277157bff9bd3eebfdf1f2c39c7c3 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 20 Jul 2020 12:58:11 +0200 Subject: [PATCH 09/15] Increate leader election lease time for machine-healthcheck The machine-api-controller components are refreshing their lease more than all other components combined. Bringing this to 90s each, will decrease etcd writes at idle. --- cmd/machine-healthcheck/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/machine-healthcheck/main.go b/cmd/machine-healthcheck/main.go index 5ad1c171d7..513df7966f 100644 --- a/cmd/machine-healthcheck/main.go +++ b/cmd/machine-healthcheck/main.go @@ -44,7 +44,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 15*time.Second, + 90*time.Second, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) From 495f3e7809125a35519dfa56070f7f3094342188 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 20 Jul 2020 13:18:22 +0200 Subject: [PATCH 10/15] Enforce default leader elect lease duration to 90s for all controllers Enforce this value in MAO deployment will ensure the value won't be overritten by changed CLI defaults in the future. --- pkg/operator/sync.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go index 4abbc1acac..d29cced53a 100644 --- a/pkg/operator/sync.go +++ b/pkg/operator/sync.go @@ -314,6 +314,7 @@ func newContainers(config *OperatorConfig, features map[string]bool) []corev1.Co "--logtostderr=true", "--v=3", "--leader-elect=true", + "--leader-elect-lease-duration=90s", fmt.Sprintf("--namespace=%s", config.TargetNamespace), } From 38d6cd2c9d9d4361c85ee037404668c3d25418a5 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Wed, 12 Aug 2020 15:10:47 +0200 Subject: [PATCH 11/15] Slow the default lease retry and renew rate for MHC controller Prevent machine controllers from writing in etcd at idle too often by setting 30s retry and 90s deadline on all renewals. BZ 1858403 --- cmd/machine-healthcheck/main.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/machine-healthcheck/main.go b/cmd/machine-healthcheck/main.go index 513df7966f..228b8cae60 100644 --- a/cmd/machine-healthcheck/main.go +++ b/cmd/machine-healthcheck/main.go @@ -17,6 +17,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) +// The default durations for the leader electrion operations. +var ( + retryPeriod = 30 * time.Second + renewDealine = 90 * time.Second +) + func printVersion() { glog.Infof("Go Version: %s", runtime.Version()) glog.Infof("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH) @@ -64,6 +70,9 @@ func main() { LeaderElectionNamespace: *leaderElectResourceNamespace, LeaderElectionID: "cluster-api-provider-healthcheck-leader", LeaseDuration: leaderElectLeaseDuration, + // Slow the default retry and renew election rate to reduce etcd writes at idle: BZ 1858400 + RetryPeriod: &retryPeriod, + RenewDeadline: &renewDealine, } if *watchNamespace != "" { From 6691a3e6a39e18b476a8b52248df97dfe158679d Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Wed, 12 Aug 2020 16:19:37 +0200 Subject: [PATCH 12/15] Slow the default lease retry and renew rate for MachineSet controller Prevent machine controllers from writing in etcd at idle too often by setting 30s retry and 90s deadline on all renewals. BZ 1858403 --- cmd/machineset/main.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/machineset/main.go b/cmd/machineset/main.go index 61bf0ecfbd..40ca045669 100644 --- a/cmd/machineset/main.go +++ b/cmd/machineset/main.go @@ -31,6 +31,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) +// The default durations for the leader electrion operations. +var ( + retryPeriod = 30 * time.Second + renewDealine = 90 * time.Second +) + func main() { flag.Set("logtostderr", "true") klog.InitFlags(nil) @@ -77,6 +83,9 @@ func main() { LeaderElectionNamespace: *leaderElectResourceNamespace, LeaderElectionID: "cluster-api-provider-machineset-leader", LeaseDuration: leaderElectLeaseDuration, + // Slow the default retry and renew election rate to reduce etcd writes at idle: BZ 1858400 + RetryPeriod: &retryPeriod, + RenewDeadline: &renewDealine, } mgr, err := manager.New(cfg, opts) if err != nil { From 408c0863e92b3676cc70717bb921aba76fb918da Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Wed, 12 Aug 2020 16:20:07 +0200 Subject: [PATCH 13/15] Slow the default lease retry and renew rate for nodelink controller Prevent machine controllers from writing in etcd at idle too often by setting 60s retry and delay on all renewals. BZ 1858403 --- cmd/nodelink-controller/main.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/nodelink-controller/main.go b/cmd/nodelink-controller/main.go index 99572c5a54..047c1f3984 100644 --- a/cmd/nodelink-controller/main.go +++ b/cmd/nodelink-controller/main.go @@ -15,6 +15,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) +// The default durations for the leader electrion operations. +var ( + retryPeriod = 30 * time.Second + renewDealine = 90 * time.Second +) + func printVersion() { klog.Infof("Go Version: %s", runtime.Version()) klog.Infof("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH) @@ -65,6 +71,9 @@ func main() { LeaderElectionNamespace: *leaderElectResourceNamespace, LeaderElectionID: "cluster-api-provider-nodelink-leader", LeaseDuration: leaderElectLeaseDuration, + // Slow the default retry and renew election rate to reduce etcd writes at idle: BZ 1858400 + RetryPeriod: &retryPeriod, + RenewDeadline: &renewDealine, } if *watchNamespace != "" { opts.Namespace = *watchNamespace From 928eec59d7c3846b80cb12cbcb139461f80c76dd Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Wed, 12 Aug 2020 16:20:25 +0200 Subject: [PATCH 14/15] Slow the default lease retry and renew rate for vsphere controller Prevent machine controllers from writing in etcd at idle too often by setting 60s retry and delay on all renewals. BZ 1858403 --- cmd/vsphere/main.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/vsphere/main.go b/cmd/vsphere/main.go index 2a8fa8e01e..9b2bc61295 100644 --- a/cmd/vsphere/main.go +++ b/cmd/vsphere/main.go @@ -18,6 +18,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) +// The default durations for the leader electrion operations. +var ( + retryPeriod = 30 * time.Second + renewDealine = 90 * time.Second +) + func main() { var printVersion bool flag.BoolVar(&printVersion, "version", false, "print version and exit") @@ -64,6 +70,9 @@ func main() { LeaderElectionNamespace: *leaderElectResourceNamespace, LeaderElectionID: "cluster-api-provider-vsphere-leader", LeaseDuration: leaderElectLeaseDuration, + // Slow the default retry and renew election rate to reduce etcd writes at idle: BZ 1858400 + RetryPeriod: &retryPeriod, + RenewDeadline: &renewDealine, } if *watchNamespace != "" { From 30486159bbf0cd1d2fb7637bde8118541e167776 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Thu, 13 Aug 2020 15:28:01 +0200 Subject: [PATCH 15/15] Reconcider the default lease durations on 120/110/90 --- cmd/machine-healthcheck/main.go | 7 ++++--- cmd/machineset/main.go | 7 ++++--- cmd/nodelink-controller/main.go | 7 ++++--- cmd/vsphere/main.go | 7 ++++--- pkg/operator/sync.go | 2 +- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/cmd/machine-healthcheck/main.go b/cmd/machine-healthcheck/main.go index 228b8cae60..b595f4c54d 100644 --- a/cmd/machine-healthcheck/main.go +++ b/cmd/machine-healthcheck/main.go @@ -19,8 +19,9 @@ import ( // The default durations for the leader electrion operations. var ( - retryPeriod = 30 * time.Second - renewDealine = 90 * time.Second + leaseDuration = 120 * time.Second + renewDealine = 110 * time.Second + retryPeriod = 90 * time.Second ) func printVersion() { @@ -50,7 +51,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 90*time.Second, + leaseDuration, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) diff --git a/cmd/machineset/main.go b/cmd/machineset/main.go index 40ca045669..2448eb3a33 100644 --- a/cmd/machineset/main.go +++ b/cmd/machineset/main.go @@ -33,8 +33,9 @@ import ( // The default durations for the leader electrion operations. var ( - retryPeriod = 30 * time.Second - renewDealine = 90 * time.Second + leaseDuration = 120 * time.Second + renewDealine = 110 * time.Second + retryPeriod = 90 * time.Second ) func main() { @@ -57,7 +58,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 90*time.Second, + leaseDuration, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) diff --git a/cmd/nodelink-controller/main.go b/cmd/nodelink-controller/main.go index 047c1f3984..28367dc825 100644 --- a/cmd/nodelink-controller/main.go +++ b/cmd/nodelink-controller/main.go @@ -17,8 +17,9 @@ import ( // The default durations for the leader electrion operations. var ( - retryPeriod = 30 * time.Second - renewDealine = 90 * time.Second + leaseDuration = 120 * time.Second + renewDealine = 110 * time.Second + retryPeriod = 90 * time.Second ) func printVersion() { @@ -50,7 +51,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 90*time.Second, + leaseDuration, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) diff --git a/cmd/vsphere/main.go b/cmd/vsphere/main.go index 9b2bc61295..689a9c99b4 100644 --- a/cmd/vsphere/main.go +++ b/cmd/vsphere/main.go @@ -20,8 +20,9 @@ import ( // The default durations for the leader electrion operations. var ( - retryPeriod = 30 * time.Second - renewDealine = 90 * time.Second + leaseDuration = 120 * time.Second + renewDealine = 110 * time.Second + retryPeriod = 90 * time.Second ) func main() { @@ -49,7 +50,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 90*time.Second, + leaseDuration, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go index d29cced53a..1355bd6bc1 100644 --- a/pkg/operator/sync.go +++ b/pkg/operator/sync.go @@ -314,7 +314,7 @@ func newContainers(config *OperatorConfig, features map[string]bool) []corev1.Co "--logtostderr=true", "--v=3", "--leader-elect=true", - "--leader-elect-lease-duration=90s", + "--leader-elect-lease-duration=120s", fmt.Sprintf("--namespace=%s", config.TargetNamespace), }