From 954124369963d55110414632c4791b4834f0b3e9 Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Tue, 31 May 2022 17:12:00 +0900 Subject: [PATCH 1/2] Use cert-manager to deploy internal certificates --- cmd/activator/main.go | 7 ++-- pkg/reconciler/autoscaling/kpa/kpa.go | 6 ---- test/config/tls/cert.yaml | 25 +++++++++++++ test/config/tls/config-network.yaml | 4 +-- test/e2e-common.sh | 15 ++++++-- test/e2e/autoscale_test.go | 11 ------ test/generate-cert.sh | 47 ------------------------- third_party/kourier-latest/kourier.yaml | 2 +- 8 files changed, 46 insertions(+), 71 deletions(-) create mode 100644 test/config/tls/cert.yaml delete mode 100755 test/generate-cert.sh diff --git a/cmd/activator/main.go b/cmd/activator/main.go index 403ef7a22fdb..d22f6de3df9c 100644 --- a/cmd/activator/main.go +++ b/cmd/activator/main.go @@ -163,7 +163,10 @@ func main() { // At this moment activator with TLS does not disable HTTP. // See also https://github.com/knative/serving/issues/12808. if tlsEnabled { - caSecret, err := kubeClient.CoreV1().Secrets(system.Namespace()).Get(ctx, networkConfig.QueueProxyCA, metav1.GetOptions{}) + // TODO: Allow to configure the namespace. + certManagerNamespace := "cert-manager" + + caSecret, err := kubeClient.CoreV1().Secrets(certManagerNamespace).Get(ctx, networkConfig.QueueProxyCA, metav1.GetOptions{}) if err != nil { logger.Fatalw("Failed to get secret", zap.Error(err)) } @@ -173,7 +176,7 @@ func main() { pool = x509.NewCertPool() } - if ok := pool.AppendCertsFromPEM(caSecret.Data["ca.crt"]); !ok { + if ok := pool.AppendCertsFromPEM(caSecret.Data["tls.crt"]); !ok { logger.Fatalw("Failed to append ca cert to the RootCAs") } diff --git a/pkg/reconciler/autoscaling/kpa/kpa.go b/pkg/reconciler/autoscaling/kpa/kpa.go index 5f38600e585e..8258bce9bd15 100644 --- a/pkg/reconciler/autoscaling/kpa/kpa.go +++ b/pkg/reconciler/autoscaling/kpa/kpa.go @@ -126,12 +126,6 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, pa *autoscalingv1alpha1. mode := nv1alpha1.SKSOperationModeProxy switch { - // When activator CA is enabled, force activator always in path. - // TODO: This is a temporary state and to be fixed. - // See also issues/11906 and issues/12797. - case len(config.FromContext(ctx).Network.ActivatorCA) > 0: - mode = nv1alpha1.SKSOperationModeProxy - // If the want == -1 and PA is inactive that implies the autoscaler // has no knowledge of the revision (due to restart) but it was previously // scaled down (inactive). In this instance we want to remain in Proxy Mode diff --git a/test/config/tls/cert.yaml b/test/config/tls/cert.yaml new file mode 100644 index 000000000000..fe505b622888 --- /dev/null +++ b/test/config/tls/cert.yaml @@ -0,0 +1,25 @@ +# Copyright 2022 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: route-test +spec: + dnsNames: + - knative + issuerRef: + kind: ClusterIssuer + name: ca-issuer + secretName: server-certs diff --git a/test/config/tls/config-network.yaml b/test/config/tls/config-network.yaml index ae048d23acbc..e6e3fe3f0596 100644 --- a/test/config/tls/config-network.yaml +++ b/test/config/tls/config-network.yaml @@ -21,9 +21,9 @@ metadata: app.kubernetes.io/version: devel serving.knative.dev/release: devel data: - activator-ca: "serving-ca" + activator-ca: "ca-key-pair" activator-san: "knative" activator-cert-secret: "server-certs" - queue-proxy-ca: "serving-ca" + queue-proxy-ca: "ca-key-pair" queue-proxy-san: "knative" queue-proxy-cert-secret: "server-certs" diff --git a/test/e2e-common.sh b/test/e2e-common.sh index a261e9c65a01..0acd9b69c814 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -359,12 +359,23 @@ function install() { fi if (( ENABLE_TLS )); then - echo "Generate certificates" - bash ${REPO_ROOT_DIR}/test/generate-cert.sh + # NOTE: cert-manager is always deployed by ytt. + + echo "Deploy CA and CA issuer" + kubectl apply -n cert-manager -f ${REPO_ROOT_DIR}/test/config/autotls/certmanager/caissuer/secret.yaml + kubectl apply -f ${REPO_ROOT_DIR}/test/config/autotls/certmanager/caissuer/issuer.yaml + + echo "Deploy Certificates into serving system and user(test) namespaces" + kubectl apply -n ${SYSTEM_NAMESPACE} -f ${REPO_ROOT_DIR}/test/config/tls/cert.yaml + kubectl apply -n serving-tests -f ${REPO_ROOT_DIR}/test/config/tls/cert.yaml + kubectl apply -n serving-tests-alt -f ${REPO_ROOT_DIR}/test/config/tls/cert.yaml + + kubectl wait --timeout=120s --for=condition=Ready certificate -n ${SYSTEM_NAMESPACE} --all echo "Patch to activator to serve TLS" kubectl apply -n ${SYSTEM_NAMESPACE} -f ${REPO_ROOT_DIR}/test/config/tls/config-network.yaml kubectl delete pod -n ${SYSTEM_NAMESPACE} -l app=activator + kubectl wait --timeout=60s --for=condition=Available deployment -n ${SYSTEM_NAMESPACE} activator fi } diff --git a/test/e2e/autoscale_test.go b/test/e2e/autoscale_test.go index af8508d4a8f8..806c7f59f785 100644 --- a/test/e2e/autoscale_test.go +++ b/test/e2e/autoscale_test.go @@ -28,7 +28,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - netcfg "knative.dev/networking/pkg/config" "knative.dev/pkg/system" "knative.dev/serving/pkg/apis/autoscaling" "knative.dev/serving/pkg/networking" @@ -133,16 +132,6 @@ func TestTargetBurstCapacity(t *testing.T) { })) test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) - cm, err := ctx.clients.KubeClient.CoreV1().ConfigMaps(system.Namespace()). - Get(context.Background(), netcfg.ConfigMapName, metav1.GetOptions{}) - if err != nil { - t.Fatal("Fail to get ConfigMap config-network:", err) - } - if cm.Data[netcfg.ActivatorCAKey] != "" { - // TODO: Remove this when https://github.com/knative/serving/issues/12797 was done. - t.Skip("Skipping TestTargetBurstCapacity as activator-ca is specified. See issue/12797.") - } - cfg, err := autoscalerCM(ctx.clients) if err != nil { t.Fatal("Error retrieving autoscaler configmap:", err) diff --git a/test/generate-cert.sh b/test/generate-cert.sh deleted file mode 100755 index 5569f166ad32..000000000000 --- a/test/generate-cert.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2022 The Knative Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-knative-serving}" -TEST_NAMESPACE=serving-tests -TEST_NAMESPACE_ALT=serving-tests-alt -out_dir="$(mktemp -d /tmp/certs-XXX)" -san="knative" - -# Generate Root key and cert. -openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=Example/CN=Example' -keyout "${out_dir}"/root.key -out "${out_dir}"/root.crt - -# Create server key -openssl req -out "${out_dir}"/tls.csr -newkey rsa:2048 -nodes -keyout "${out_dir}"/tls.key -subj "/CN=Example/O=Example" -addext "subjectAltName = DNS:$san" - -# Create server certs -openssl x509 -req -extfile <(printf "subjectAltName=DNS:$san") -days 365 -in "${out_dir}"/tls.csr -CA "${out_dir}"/root.crt -CAkey "${out_dir}"/root.key -CAcreateserial -out "${out_dir}"/tls.crt - -# Create secret -kubectl create -n ${SYSTEM_NAMESPACE} secret generic serving-ca \ - --from-file=ca.crt="${out_dir}"/root.crt --dry-run=client -o yaml | kubectl apply -f - - -kubectl create -n ${SYSTEM_NAMESPACE} secret tls server-certs \ - --key="${out_dir}"/tls.key \ - --cert="${out_dir}"/tls.crt --dry-run=client -o yaml | kubectl apply -f - - -# Create secrets for test namespaces -kubectl create -n ${TEST_NAMESPACE} secret tls server-certs \ - --key="${out_dir}"/tls.key \ - --cert="${out_dir}"/tls.crt --dry-run=client -o yaml | kubectl apply -f - - -kubectl create -n ${TEST_NAMESPACE_ALT} secret tls server-certs \ - --key="${out_dir}"/tls.key \ - --cert="${out_dir}"/tls.crt --dry-run=client -o yaml | kubectl apply -f - diff --git a/third_party/kourier-latest/kourier.yaml b/third_party/kourier-latest/kourier.yaml index 8c08ae521fbd..3b88e113f0ec 100644 --- a/third_party/kourier-latest/kourier.yaml +++ b/third_party/kourier-latest/kourier.yaml @@ -299,7 +299,7 @@ spec: app: net-kourier-controller spec: containers: - - image: gcr.io/knative-nightly/knative.dev/net-kourier/cmd/kourier@sha256:9209fa7b69c3d536f940768c5f6c8975a6214db5b814b807bc3d23bb6b27dc96 + - image: gcr.io/gcp-compute-engine-223401/kourier-b74c3918b7eee585f87df62ccd297dc8:latest name: controller env: - name: CERTS_SECRET_NAMESPACE From 5b0beb3028cdac039b80a206a3b095ebddbd21f7 Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Mon, 6 Jun 2022 23:15:38 +0900 Subject: [PATCH 2/2] Drop utest for activator always in the path Drop unit test added by 21c05dc9d9a4a89298e12425e7c32cbb1ef1adf9 --- pkg/reconciler/autoscaling/kpa/kpa_test.go | 32 ---------------------- 1 file changed, 32 deletions(-) diff --git a/pkg/reconciler/autoscaling/kpa/kpa_test.go b/pkg/reconciler/autoscaling/kpa/kpa_test.go index 3d876cc106aa..516c05260a6d 100644 --- a/pkg/reconciler/autoscaling/kpa/kpa_test.go +++ b/pkg/reconciler/autoscaling/kpa/kpa_test.go @@ -1141,38 +1141,6 @@ func TestReconcile(t *testing.T) { WithPAMetricsService(privateSvc), WithObservedGeneration(1), ), }}, - }, { - Name: "we have enough burst capacity, but keep proxy mode as activator CA is enabled", - Key: key, - Ctx: context.WithValue(context.WithValue(context.Background(), netConfigKey{}, activatorCertsNetConfig()), deciderKey{}, - decider(testNamespace, testRevision, defaultScale, /* desiredScale */ - 1 /* ebc */)), - Objects: []runtime.Object{ - kpa(testNamespace, testRevision, WithPASKSReady, WithTraffic, markScaleTargetInitialized, - WithPAMetricsService(privateSvc), withScales(1, defaultScale), - WithPAStatusService(testRevision), WithObservedGeneration(1)), - defaultProxySKS, - metric(testNamespace, testRevision), - defaultDeployment, - defaultReady}, - // No update from ProxySKS. - }, { - Name: "we have enough burst capacity, but switch to keep proxy mode as activator CA is turned on", - Key: key, - Ctx: context.WithValue(context.WithValue(context.Background(), netConfigKey{}, activatorCertsNetConfig()), deciderKey{}, - decider(testNamespace, testRevision, defaultScale, /* desiredScale */ - 1 /* ebc */)), - Objects: []runtime.Object{ - kpa(testNamespace, testRevision, WithPASKSReady, WithTraffic, markScaleTargetInitialized, - WithPAMetricsService(privateSvc), withScales(1, defaultScale), - WithPAStatusService(testRevision), WithObservedGeneration(1)), - defaultSKS, - metric(testNamespace, testRevision), - defaultDeployment, - defaultReady}, - WantUpdates: []clientgotesting.UpdateActionImpl{{ - Object: defaultProxySKS, - }}, }} table.Test(t, MakeFactory(func(ctx context.Context, listers *Listers, cmw configmap.Watcher) controller.Reconciler {