From 2c383f93698f1bc117a9984113f03ffbec1387f7 Mon Sep 17 00:00:00 2001 From: Dave Protasowski Date: Thu, 30 Jul 2020 09:53:02 -0400 Subject: [PATCH 1/2] bump webhook resources since we're seeing EOFs --- config/core/deployments/webhook.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/core/deployments/webhook.yaml b/config/core/deployments/webhook.yaml index 61015fc5fd2..ee2febb2380 100644 --- a/config/core/deployments/webhook.yaml +++ b/config/core/deployments/webhook.yaml @@ -42,12 +42,12 @@ spec: resources: requests: # taken from serving. - cpu: 20m - memory: 20Mi + cpu: 100m + memory: 100Mi limits: # taken from serving. - cpu: 200m - memory: 200Mi + cpu: 500m + memory: 500Mi env: - name: SYSTEM_NAMESPACE From 9b5309ae6b78f113332804b67fca29a62fff6b8c Mon Sep 17 00:00:00 2001 From: Dave Protasowski Date: Fri, 31 Jul 2020 15:10:21 -0400 Subject: [PATCH 2/2] instrument --- vendor/knative.dev/pkg/webhook/webhook.go | 24 +++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/vendor/knative.dev/pkg/webhook/webhook.go b/vendor/knative.dev/pkg/webhook/webhook.go index 8d628c5e32e..b61bbb171c4 100644 --- a/vendor/knative.dev/pkg/webhook/webhook.go +++ b/vendor/knative.dev/pkg/webhook/webhook.go @@ -21,7 +21,9 @@ import ( "crypto/tls" "errors" "fmt" + "net" "net/http" + "sync/atomic" "time" // Injection stuff @@ -174,14 +176,29 @@ func (wh *Webhook) Run(stop <-chan struct{}) error { logger := wh.Logger ctx := logging.WithLogger(context.Background(), logger) + count := int32(0) + drainer := &handlers.Drainer{ Inner: wh, QuietPeriod: wh.gracePeriod, } + counter := func(res http.ResponseWriter, req *http.Request) { + c := atomic.AddInt32(&count, 1) + wh.Logger.Infof("request count (+) %d ", c) + + drainer.ServeHTTP(res, req) + + c = atomic.AddInt32(&count, -1) + wh.Logger.Infof("request count (-) %d ", c) + } + server := &http.Server{ - Handler: drainer, + Handler: http.HandlerFunc(counter), Addr: fmt.Sprintf(":%d", wh.Options.Port), + ConnState: func(c net.Conn, state http.ConnState) { + wh.Logger.Infof("conn state %s %s", c.RemoteAddr(), state.String()) + }, TLSConfig: &tls.Config{ GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { secret, err := wh.secretlister.Secrets(system.Namespace()).Get(wh.Options.SecretName) @@ -224,8 +241,11 @@ func (wh *Webhook) Run(stop <-chan struct{}) error { // Start failing readiness probes immediately. logger.Info("Starting to fail readiness probes...") drainer.Drain() + logger.Info("Draining finished") - return server.Shutdown(context.Background()) + err := server.Shutdown(context.Background()) + logger.Info("Shutdown finished") + return err }) // Wait for all outstanding go routined to terminate, including our new one.