Add leader election

cmd/webhook/main.go

@@ -23,6 +23,7 @@ import (
 	"knative.dev/pkg/configmap"
 	"knative.dev/pkg/controller"
 	"knative.dev/pkg/injection/sharedmain"
+	pkgleaderelection "knative.dev/pkg/leaderelection"
 	"knative.dev/pkg/logging"
 	"knative.dev/pkg/metrics"
 	"knative.dev/pkg/signals"
@@ -41,6 +42,7 @@ import (
 	v1 "knative.dev/serving/pkg/apis/serving/v1"
 	"knative.dev/serving/pkg/apis/serving/v1alpha1"
 	"knative.dev/serving/pkg/apis/serving/v1beta1"
+	"knative.dev/serving/pkg/leaderelection"
 
 	// config validation constructors
 	tracingconfig "knative.dev/pkg/tracing/config"
@@ -139,17 +141,18 @@ func NewConfigValidationController(ctx context.Context, cmw configmap.Watcher) *
 
 		// The configmaps to validate.
 		configmap.Constructors{
-			tracingconfig.ConfigName:         tracingconfig.NewTracingConfigFromConfigMap,
-			autoscalerconfig.ConfigName:      autoscalerconfig.NewConfigFromConfigMap,
-			certconfig.CertManagerConfigName: certconfig.NewCertManagerConfigFromConfigMap,
-			gc.ConfigName:                    gc.NewConfigFromConfigMapFunc(ctx),
-			network.ConfigName:               network.NewConfigFromConfigMap,
-			istioconfig.IstioConfigName:      istioconfig.NewIstioFromConfigMap,
-			deployment.ConfigName:            deployment.NewConfigFromConfigMap,
-			metrics.ConfigMapName():          metrics.NewObservabilityConfigFromConfigMap,
-			logging.ConfigMapName():          logging.NewConfigFromConfigMap,
-			domainconfig.DomainConfigName:    domainconfig.NewDomainFromConfigMap,
-			defaultconfig.DefaultsConfigName: defaultconfig.NewDefaultsConfigFromConfigMap,
+			tracingconfig.ConfigName:          tracingconfig.NewTracingConfigFromConfigMap,
+			autoscalerconfig.ConfigName:       autoscalerconfig.NewConfigFromConfigMap,
+			certconfig.CertManagerConfigName:  certconfig.NewCertManagerConfigFromConfigMap,
+			gc.ConfigName:                     gc.NewConfigFromConfigMapFunc(ctx),
+			network.ConfigName:                network.NewConfigFromConfigMap,
+			istioconfig.IstioConfigName:       istioconfig.NewIstioFromConfigMap,
+			deployment.ConfigName:             deployment.NewConfigFromConfigMap,
+			metrics.ConfigMapName():           metrics.NewObservabilityConfigFromConfigMap,
+			logging.ConfigMapName():           logging.NewConfigFromConfigMap,
+			pkgleaderelection.ConfigMapName(): leaderelection.ValidateConfig,
+			domainconfig.DomainConfigName:     domainconfig.NewDomainFromConfigMap,
+			defaultconfig.DefaultsConfigName:  defaultconfig.NewDefaultsConfigFromConfigMap,
 		},
 	)
 }

config/config-leader-election.yaml

@@ -0,0 +1 @@
+core/configmaps/leader-election.yaml

config/core/configmaps/leader-election.yaml

@@ -0,0 +1,69 @@
+# Copyright 2020 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-leader-election
+  namespace: knative-serving
+  labels:
+    serving.knative.dev/release: devel
+data:
+  # An inactive but valid configuration follows; see example.
+  resourceLock: "leases"
+  leaseDuration: "15s"
+  renewDeadline: "10s"
+  retryPeriod: "2s"
+  _example: |
+    ################################
+    #                              #
+    #    EXAMPLE CONFIGURATION     #
+    #                              #
+    ################################
+
+    # This block is not actually functional configuration,
+    # but serves to illustrate the available configuration
+    # options and document them in a way that is accessible
+    # to users that `kubectl edit` this config map.
+    #
+    # These sample configuration options may be copied out of
+    # this example block and unindented to be in the data block
+    # to actually change the configuration.
+
+    # resourceLock controls which API resource is used as the basis for the
+    # leader election lock. Valid values are:
+    #
+    # - leases -> use the coordination API
+    # - configmaps -> use configmaps
+    # - endpoints -> use endpoints
+    resourceLock: "leases"
+
+    # leaseDuration is how long non-leaders will wait to try to acquire the
+    # lock; 15 seconds is the value used by core kubernetes controllers.
+    leaseDuration: "15s"
+    # renewDeadline is how long a leader will try to renew the lease before
+    # giving up; 10 seconds is the value used by core kubernetes controllers.
+    renewDeadline: "10s"
+    # retryPeriod is how long the leader election client waits between tries of
+    # actions; 2 seconds is the value used by core kubernetes controllers.
+    retryPeriod: "2s"
+    # enabledComponents is a comma-delimited list of component names for which
+    # leader election is enabled. Valid values are:
+    #
+    # - controller
+    # - hpaautoscaler
+    # - certcontroller
+    # - istiocontroller
+    # - nscontroller
+    enabledComponents: "controller,hpaautoscaler,certcontroller,istiocontroller,nscontroller"

config/core/roles/clusterrole.yaml

@@ -38,6 +38,9 @@ rules:
   - apiGroups: ["autoscaling"]
     resources: ["horizontalpodautoscalers"]
     verbs: ["get", "list", "create", "update", "delete", "patch", "watch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "create", "update", "delete", "patch", "watch"]
   - apiGroups: ["serving.knative.dev", "autoscaling.internal.knative.dev", "networking.internal.knative.dev"]
     resources: ["*", "*/status", "*/finalizers"]
     verbs: ["get", "list", "create", "update", "delete", "deletecollection", "patch", "watch"]

pkg/leaderelection/config.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2020 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package leaderelection
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	kle "knative.dev/pkg/leaderelection"
+)
+
+var (
+	validComponents = sets.NewString(
+		"controller",
+		"hpaautoscaler",
+		"certcontroller",
+		"istiocontroller",
+		"nscontroller")
+)
+
+// ValidateConfig enriches the leader election config validation
+// with extra validations specific to serving.
+func ValidateConfig(configMap *corev1.ConfigMap) (*kle.Config, error) {
+	config, err := kle.NewConfigFromMap(configMap.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, component := range config.EnabledComponents.List() {
+		if !validComponents.Has(component) {
+			return nil, fmt.Errorf("invalid enabledComponent %q: valid values are %q", component, validComponents.List())
+		}
+	}
+
+	return config, nil
+}

pkg/leaderelection/config_test.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2020 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package leaderelection
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	kle "knative.dev/pkg/leaderelection"
+)
+
+func okConfig() *kle.Config {
+	return &kle.Config{
+		ResourceLock:      "leases",
+		LeaseDuration:     15 * time.Second,
+		RenewDeadline:     10 * time.Second,
+		RetryPeriod:       2 * time.Second,
+		EnabledComponents: sets.NewString("controller"),
+	}
+}
+
+func okData() map[string]string {
+	return map[string]string{
+		"resourceLock": "leases",
+		// values in this data come from the defaults suggested in the
+		// code:
+		// https://github.com/kubernetes/client-go/blob/kubernetes-1.16.0/tools/leaderelection/leaderelection.go
+		"leaseDuration":     "15s",
+		"renewDeadline":     "10s",
+		"retryPeriod":       "2s",
+		"enabledComponents": "controller",
+	}
+}
+
+func TestValidateConfig(t *testing.T) {
+	cases := []struct {
+		name     string
+		data     map[string]string
+		expected *kle.Config
+		err      error
+	}{
+		{
+			name:     "OK",
+			data:     okData(),
+			expected: okConfig(),
+		},
+		{
+			name: "invalid component",
+			data: func() map[string]string {
+				data := okData()
+				data["enabledComponents"] = "controller,frobulator"
+				return data
+			}(),
+			err: errors.New(`invalid enabledComponent "frobulator": valid values are ["certcontroller" "controller" "hpaautoscaler" "istiocontroller" "nscontroller"]`),
+		},
+	}
+
+	for i := range cases {
+		tc := cases[i]
+		actualConfig, actualErr := ValidateConfig(&corev1.ConfigMap{Data: tc.data})
+		if !reflect.DeepEqual(tc.err, actualErr) {
+			t.Errorf("%v: expected error %v, got %v", tc.name, tc.err, actualErr)
+			continue
+		}
+
+		if !reflect.DeepEqual(tc.expected, actualConfig) {
+			t.Errorf("%v: expected config:\n%+v\ngot:\n%+v", tc.name, tc.expected, actualConfig)
+			continue
+		}
+	}
+
+}