diff --git a/Gopkg.lock b/Gopkg.lock index a9224673..27ba4c0f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1413,6 +1413,7 @@ "k8s.io/apimachinery/pkg/runtime/serializer", "k8s.io/apimachinery/pkg/types", "k8s.io/apimachinery/pkg/util/runtime", + "k8s.io/apimachinery/pkg/util/sets", "k8s.io/apimachinery/pkg/util/wait", "k8s.io/apimachinery/pkg/watch", "k8s.io/client-go/discovery", diff --git a/config/300-serving-v1alpha1-knativeserving-crd.yaml b/config/300-serving-v1alpha1-knativeserving-crd.yaml index 87b510e8..5cf6730a 100644 --- a/config/300-serving-v1alpha1-knativeserving-crd.yaml +++ b/config/300-serving-v1alpha1-knativeserving-crd.yaml @@ -119,6 +119,14 @@ spec: name: description: The name of the ConfigMap or Secret type: string + high-availability: + description: Allows specification of HA control plane + type: object + properties: + replicas: + description: The number of replicas that HA parts of the control plane will be scaled to + type: integer + minimum: 1 type: object status: description: Status defines the observed state of KnativeServing diff --git a/pkg/apis/serving/v1alpha1/knativeserving_types.go b/pkg/apis/serving/v1alpha1/knativeserving_types.go index 80065546..a45cc334 100644 --- a/pkg/apis/serving/v1alpha1/knativeserving_types.go +++ b/pkg/apis/serving/v1alpha1/knativeserving_types.go @@ -68,6 +68,15 @@ type CustomCerts struct { Name string `json:"name"` } +// HighAvailability specifies options for deploying Knative Serving control +// plane in a highly available manner. Note that HighAvailability is still in +// progress and does not currently provide a completely HA control plane. +type HighAvailability struct { + // Replicas is the number of replicas that HA parts of the control plane + // will be scaled to. + Replicas int32 `json:"replicas"` +} + // KnativeServingSpec defines the desired state of KnativeServing // +k8s:openapi-gen=true type KnativeServingSpec struct { @@ -92,6 +101,10 @@ type KnativeServingSpec struct { // Enables controller to trust registries with self-signed certificates ControllerCustomCerts CustomCerts `json:"controller-custom-certs,omitempty"` + + // Allows specification of HA control plane + // +optional + HighAvailability *HighAvailability `json:"high-availability,omitempty"` } // KnativeServingStatus defines the observed state of KnativeServing diff --git a/pkg/apis/serving/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/serving/v1alpha1/zz_generated.deepcopy.go index 4249c311..85e0bcdd 100644 --- a/pkg/apis/serving/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/serving/v1alpha1/zz_generated.deepcopy.go @@ -42,6 +42,22 @@ func (in *CustomCerts) DeepCopy() *CustomCerts { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HighAvailability) DeepCopyInto(out *HighAvailability) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HighAvailability. +func (in *HighAvailability) DeepCopy() *HighAvailability { + if in == nil { + return nil + } + out := new(HighAvailability) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IstioGatewayOverride) DeepCopyInto(out *IstioGatewayOverride) { *out = *in @@ -150,6 +166,11 @@ func (in *KnativeServingSpec) DeepCopyInto(out *KnativeServingSpec) { in.KnativeIngressGateway.DeepCopyInto(&out.KnativeIngressGateway) in.ClusterLocalGateway.DeepCopyInto(&out.ClusterLocalGateway) out.ControllerCustomCerts = in.ControllerCustomCerts + if in.HighAvailability != nil { + in, out := &in.HighAvailability, &out.HighAvailability + *out = new(HighAvailability) + **out = **in + } return } diff --git a/pkg/reconciler/knativeserving/common/extensions.go b/pkg/reconciler/knativeserving/common/extensions.go index 9398e6b9..6138d1d7 100644 --- a/pkg/reconciler/knativeserving/common/extensions.go +++ b/pkg/reconciler/knativeserving/common/extensions.go @@ -40,6 +40,7 @@ func (platforms Platforms) Transformers(kubeClientSet kubernetes.Interface, inst ImageTransform(instance, log), GatewayTransform(instance, log), CustomCertsTransform(instance, log), + HighAvailabilityTransform(instance, log), } for _, fn := range platforms { transformer, err := fn(kubeClientSet, log) diff --git a/pkg/reconciler/knativeserving/common/ha.go b/pkg/reconciler/knativeserving/common/ha.go new file mode 100644 index 00000000..2cee5953 --- /dev/null +++ b/pkg/reconciler/knativeserving/common/ha.go @@ -0,0 +1,74 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + mf "github.com/manifestival/manifestival" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" + servingv1alpha1 "knative.dev/serving-operator/pkg/apis/serving/v1alpha1" +) + +const ( + configMapName = "config-leader-election" + enabledComponentsKey = "enabledComponents" + componentsValue = "controller,hpaautoscaler,certcontroller,istiocontroller,nscontroller" +) + +var deploymentNames = sets.NewString( + "controller", + "autoscaler-hpa", + "networking-certmanager", + "networking-ns-cert", + "networking-istio", +) + +// HighAvailabilityTransform mutates configmaps and replicacounts of certain +// controllers when HA control plane is specified. +func HighAvailabilityTransform(instance *servingv1alpha1.KnativeServing, log *zap.SugaredLogger) mf.Transformer { + return func(u *unstructured.Unstructured) error { + if instance.Spec.HighAvailability == nil { + return nil + } + + // Transform the leader election config. + if u.GetKind() == "ConfigMap" && u.GetName() == "config-leader-election" { + data, ok, err := unstructured.NestedStringMap(u.UnstructuredContent(), "data") + if err != nil { + return nil + } + if !ok { + data = map[string]string{} + } + + data[enabledComponentsKey] = componentsValue + if err := unstructured.SetNestedStringMap(u.Object, data, "data"); err != nil { + return err + } + } + + // Transform deployments that support HA. + if u.GetKind() == "Deployment" && deploymentNames.Has(u.GetName()) { + if err := unstructured.SetNestedField(u.Object, int64(instance.Spec.HighAvailability.Replicas), "spec", "replicas"); err != nil { + return err + } + } + + return nil + } +} diff --git a/pkg/reconciler/knativeserving/common/ha_test.go b/pkg/reconciler/knativeserving/common/ha_test.go new file mode 100644 index 00000000..24768357 --- /dev/null +++ b/pkg/reconciler/knativeserving/common/ha_test.go @@ -0,0 +1,150 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/kubernetes/scheme" + + servingv1alpha1 "knative.dev/serving-operator/pkg/apis/serving/v1alpha1" +) + +func TestHighAvailabilityTransform(t *testing.T) { + cases := []struct { + name string + config *servingv1alpha1.HighAvailability + in *unstructured.Unstructured + expected *unstructured.Unstructured + err error + }{ + { + name: "No HA; ConfigMap", + config: nil, + in: makeUnstructuredConfigMap(t, nil), + expected: makeUnstructuredConfigMap(t, nil), + }, + { + name: "HA; ConfigMap", + config: makeHa(2), + in: makeUnstructuredConfigMap(t, nil), + expected: makeUnstructuredConfigMap(t, map[string]string{ + enabledComponentsKey: componentsValue, + }), + }, + { + name: "HA; controller", + config: makeHa(2), + in: makeUnstructuredDeployment(t, "controller"), + expected: makeUnstructuredDeploymentReplicas(t, "controller", 2), + }, + { + name: "HA; autoscaler-hpa", + config: makeHa(2), + in: makeUnstructuredDeployment(t, "autoscaler-hpa"), + expected: makeUnstructuredDeploymentReplicas(t, "autoscaler-hpa", 2), + }, + { + name: "HA; networking-certmanager", + config: makeHa(2), + in: makeUnstructuredDeployment(t, "networking-certmanager"), + expected: makeUnstructuredDeploymentReplicas(t, "networking-certmanager", 2), + }, + { + name: "HA; networking-ns-cert", + config: makeHa(2), + in: makeUnstructuredDeployment(t, "networking-ns-cert"), + expected: makeUnstructuredDeploymentReplicas(t, "networking-ns-cert", 2), + }, + { + name: "HA; networking-istio", + config: makeHa(2), + in: makeUnstructuredDeployment(t, "networking-istio"), + expected: makeUnstructuredDeploymentReplicas(t, "networking-istio", 2), + }, + { + name: "HA; some-unsupported-controller", + config: makeHa(2), + in: makeUnstructuredDeployment(t, "some-unsupported-controller"), + expected: makeUnstructuredDeployment(t, "some-unsupported-controller"), + }, + } + + for i := range cases { + tc := cases[i] + + instance := &servingv1alpha1.KnativeServing{ + Spec: servingv1alpha1.KnativeServingSpec{ + HighAvailability: tc.config, + }, + } + + haTransform := HighAvailabilityTransform(instance, log) + err := haTransform(tc.in) + + assertDeepEqual(t, err, tc.err) + assertDeepEqual(t, tc.in, tc.expected) + } +} + +func makeHa(replicas int32) *servingv1alpha1.HighAvailability { + return &servingv1alpha1.HighAvailability{ + Replicas: replicas, + } +} + +func makeUnstructuredConfigMap(t *testing.T, data map[string]string) *unstructured.Unstructured { + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + }, + } + cm.Data = data + result := &unstructured.Unstructured{} + err := scheme.Scheme.Convert(cm, result, nil) + if err != nil { + t.Fatalf("Could not creat unstructured ConfigMap: %v, err: %v", cm, err) + } + + return result +} + +func makeUnstructuredDeployment(t *testing.T, name string) *unstructured.Unstructured { + return makeUnstructuredDeploymentReplicas(t, name, 1) +} + +func makeUnstructuredDeploymentReplicas(t *testing.T, name string, replicas int32) *unstructured.Unstructured { + d := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + }, + } + result := &unstructured.Unstructured{} + err := scheme.Scheme.Convert(d, result, nil) + if err != nil { + t.Fatalf("Could not creat unstructured Deployment: %v, err: %v", d, err) + } + + return result +} diff --git a/pkg/reconciler/knativeserving/common/images.go b/pkg/reconciler/knativeserving/common/images.go index 55593667..b39627a2 100644 --- a/pkg/reconciler/knativeserving/common/images.go +++ b/pkg/reconciler/knativeserving/common/images.go @@ -95,7 +95,7 @@ func updateDaemonSet(instance *servingv1alpha1.KnativeServing, u *unstructured.U func updateRegistry(spec *corev1.PodSpec, instance *servingv1alpha1.KnativeServing, log *zap.SugaredLogger, name string) { registry := instance.Spec.Registry - log.Debugw("Updating ", "name", name, "registry", registry) + log.Debugw("Updating", "name", name, "registry", registry) updateImage(spec, ®istry, log, name) spec.ImagePullSecrets = addImagePullSecrets(