Tune leader election

main.go

@@ -40,6 +40,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -77,8 +78,9 @@ const (
 )
 
 var (
-	scheme   = runtime.NewScheme()
-	setupLog = ctrl.Log.WithName("api-manager")
+	scheme              = runtime.NewScheme()
+	setupLog            = ctrl.Log.WithName("api-manager")
+	leaderRetryDuration = 5 * time.Second
 )
 
 func init() {
@@ -137,6 +139,7 @@ func main() {
 	var pvcLabelSyncWorkers int
 	var enablePVCLabelSync bool
 	var enableNodeLabelSync bool
+	var leaderRenewSeconds uint
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
@@ -179,6 +182,7 @@ func main() {
 	flag.IntVar(&pvcLabelSyncWorkers, "pvc-label-sync-workers", 5, "Maximum concurrent PVC label sync operations.")
 	flag.BoolVar(&enablePVCLabelSync, "enable-pvc-label-sync", true, "Enable pvc label sync controller.")
 	flag.BoolVar(&enableNodeLabelSync, "enable-node-label-sync", true, "Enable node label sync controller.")
+	flag.UintVar(&leaderRenewSeconds, "leader-renew-seconds", 10, "Leader renewal frequency.")
 
 	loggerOpts.BindFlags(flag.CommandLine)
 	flag.Parse()
@@ -234,14 +238,21 @@ func main() {
 		}
 	}
 
+	renewDeadline := time.Duration(leaderRenewSeconds) * time.Second
+	leaseDuration := time.Duration(int(1.2*float64(leaderRenewSeconds))) * time.Second
+
 	// Only attempt to grab leader lock once we have an API connection.
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:                  scheme,
-		MetricsBindAddress:      metricsAddr,
-		Port:                    9443,
-		LeaderElection:          enableLeaderElection,
-		LeaderElectionID:        "storageos-api-manager-leader",
-		LeaderElectionNamespace: namespace,
+		Scheme:                     scheme,
+		MetricsBindAddress:         metricsAddr,
+		Port:                       9443,
+		LeaderElection:             enableLeaderElection,
+		LeaderElectionID:           "storageos-api-manager-leader",
+		LeaderElectionNamespace:    namespace,
+		LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
+		RenewDeadline:              &renewDeadline,
+		LeaseDuration:              &leaseDuration,
+		RetryPeriod:                &leaderRetryDuration,
 	})
 	if err != nil {
 		fatal(err, "unable to start manager")