From 0517d3d8efa074fdb9bccfe62fff777cf72e8088 Mon Sep 17 00:00:00 2001 From: Annie Fu Date: Wed, 10 Jul 2019 17:45:52 -0700 Subject: [PATCH] Export webhook metrics to prometheus --- cmd/webhook/main.go | 54 ++++++++++--------- .../100-prometheus-scrape-config.yaml | 18 +++++++ config/webhook.yaml | 7 +++ 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 660dd2be04d0..523d6a7a2c37 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -35,6 +35,7 @@ import ( "knative.dev/pkg/configmap" "knative.dev/pkg/logging" "knative.dev/pkg/logging/logkey" + "knative.dev/pkg/metrics" "knative.dev/pkg/signals" "knative.dev/pkg/system" "knative.dev/pkg/version" @@ -85,6 +86,9 @@ func main() { // Watch the logging config map and dynamically update logging levels. configMapWatcher := configmap.NewInformedWatcher(kubeClient, system.Namespace()) + // Watch the observability config map and dynamically update metrics exporter. + configMapWatcher.Watch(metrics.ConfigMapName(), metrics.UpdateExporterFromConfigMap(component, logger)) + // Watch the observability config map and dynamically update request logs. configMapWatcher.Watch(logging.ConfigMapName(), logging.UpdateLevelFromConfigMap(logger, atomicLevel, component)) store := apiconfig.NewStore(logger.Named("config-store")) @@ -102,31 +106,33 @@ func main() { SecretName: "webhook-certs", WebhookName: "webhook.serving.knative.dev", } - controller := webhook.AdmissionController{ - Client: kubeClient, - Options: options, - Handlers: map[schema.GroupVersionKind]webhook.GenericCRD{ - v1alpha1.SchemeGroupVersion.WithKind("Revision"): &v1alpha1.Revision{}, - v1alpha1.SchemeGroupVersion.WithKind("Configuration"): &v1alpha1.Configuration{}, - v1alpha1.SchemeGroupVersion.WithKind("Route"): &v1alpha1.Route{}, - v1alpha1.SchemeGroupVersion.WithKind("Service"): &v1alpha1.Service{}, - v1beta1.SchemeGroupVersion.WithKind("Revision"): &v1beta1.Revision{}, - v1beta1.SchemeGroupVersion.WithKind("Configuration"): &v1beta1.Configuration{}, - v1beta1.SchemeGroupVersion.WithKind("Route"): &v1beta1.Route{}, - v1beta1.SchemeGroupVersion.WithKind("Service"): &v1beta1.Service{}, - autoscalingv1alpha1.SchemeGroupVersion.WithKind("PodAutoscaler"): &autoscalingv1alpha1.PodAutoscaler{}, - net.SchemeGroupVersion.WithKind("Certificate"): &net.Certificate{}, - net.SchemeGroupVersion.WithKind("ClusterIngress"): &net.ClusterIngress{}, - net.SchemeGroupVersion.WithKind("ServerlessService"): &net.ServerlessService{}, - }, - Logger: logger, - DisallowUnknownFields: true, - - // Decorate contexts with the current state of the config. - WithContext: func(ctx context.Context) context.Context { - return v1beta1.WithUpgradeViaDefaulting(store.ToContext(ctx)) - }, + + handlers := map[schema.GroupVersionKind]webhook.GenericCRD{ + v1alpha1.SchemeGroupVersion.WithKind("Revision"): &v1alpha1.Revision{}, + v1alpha1.SchemeGroupVersion.WithKind("Configuration"): &v1alpha1.Configuration{}, + v1alpha1.SchemeGroupVersion.WithKind("Route"): &v1alpha1.Route{}, + v1alpha1.SchemeGroupVersion.WithKind("Service"): &v1alpha1.Service{}, + v1beta1.SchemeGroupVersion.WithKind("Revision"): &v1beta1.Revision{}, + v1beta1.SchemeGroupVersion.WithKind("Configuration"): &v1beta1.Configuration{}, + v1beta1.SchemeGroupVersion.WithKind("Route"): &v1beta1.Route{}, + v1beta1.SchemeGroupVersion.WithKind("Service"): &v1beta1.Service{}, + autoscalingv1alpha1.SchemeGroupVersion.WithKind("PodAutoscaler"): &autoscalingv1alpha1.PodAutoscaler{}, + net.SchemeGroupVersion.WithKind("Certificate"): &net.Certificate{}, + net.SchemeGroupVersion.WithKind("ClusterIngress"): &net.ClusterIngress{}, + net.SchemeGroupVersion.WithKind("ServerlessService"): &net.ServerlessService{}, + } + + // Decorate contexts with the current state of the config. + ctxFunc := func(ctx context.Context) context.Context { + return v1beta1.WithUpgradeViaDefaulting(store.ToContext(ctx)) + } + + controller, err := webhook.NewAdmissionController(kubeClient, options, handlers, logger, ctxFunc, true) + + if err != nil { + logger.Fatalw("Failed to create admission controller", zap.Error(err)) } + if err = controller.Run(stopCh); err != nil { logger.Fatalw("Failed to start the admission controller", zap.Error(err)) } diff --git a/config/monitoring/metrics/prometheus/100-prometheus-scrape-config.yaml b/config/monitoring/metrics/prometheus/100-prometheus-scrape-config.yaml index 004ffa0b4d81..96c3f270e661 100644 --- a/config/monitoring/metrics/prometheus/100-prometheus-scrape-config.yaml +++ b/config/monitoring/metrics/prometheus/100-prometheus-scrape-config.yaml @@ -81,6 +81,24 @@ data: target_label: pod - source_labels: [__meta_kubernetes_service_name] target_label: service + # Webhook pods + - job_name: webhook + scrape_interval: 3s + scrape_timeout: 3s + kubernetes_sd_configs: + - role: pod + relabel_configs: + # Scrape only the the targets matching the following metadata + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_role, __meta_kubernetes_pod_container_port_name] + action: keep + regex: knative-serving;webhook;metrics-port + # Rename metadata labels to be reader friendly + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_service_name] + target_label: service # Queue proxy metrics - job_name: queue-proxy scrape_interval: 3s diff --git a/config/webhook.yaml b/config/webhook.yaml index 39b8bbcc123c..154d6507e5de 100644 --- a/config/webhook.yaml +++ b/config/webhook.yaml @@ -41,6 +41,9 @@ spec: # This is the Go import path for the binary that is containerized # and substituted here. image: github.com/knative/serving/cmd/webhook + ports: + - name: metrics-port + containerPort: 9090 resources: # Request 2x what we saw running e2e requests: @@ -60,6 +63,10 @@ spec: fieldPath: metadata.namespace - name: CONFIG_LOGGING_NAME value: config-logging + - name: CONFIG_OBSERVABILITY_NAME + value: config-observability + - name: METRICS_DOMAIN + value: knative.dev/serving securityContext: allowPrivilegeEscalation: false volumes: