From c6acdcb697421d8f817a3160c4a4216333c3a1b2 Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Tue, 5 Apr 2022 21:03:37 +0900 Subject: [PATCH 1/3] Add `NewProxyAutoTLSTransport` and `DialTLSWithBackOff` to support TLS proxy Part of: https://github.com/knative/serving/issues/12503 PoC: https://github.com/knative/serving/pull/12815 This patch `NewProxyAutoTLSTransport` which is `NewProxyAutoTransport + TLS config. Current proxy does not support TLS but it needs for https://github.com/knative/serving/issues/12503. `DialTLSWithBackOff` is also `DialWithBackOff` + TLS config. It needs `newH2Transport` which handles HTTP2 with TLS. --- network/h2c.go | 13 +++++++++++ network/transports.go | 45 +++++++++++++++++++++++++++++++++++--- network/transports_test.go | 41 +++++++++++++++++++++++++++++++++- 3 files changed, 95 insertions(+), 4 deletions(-) diff --git a/network/h2c.go b/network/h2c.go index f950b9c34f..6cc0fa733d 100644 --- a/network/h2c.go +++ b/network/h2c.go @@ -54,3 +54,16 @@ func newH2CTransport(disableCompression bool) http.RoundTripper { }, } } + +// newH2Transport constructs a neew H2 transport. That transport will handles HTTPS traffic +// with TLS config. +func newH2Transport(disableCompression bool, tlsConf *tls.Config) http.RoundTripper { + return &http2.Transport{ + DisableCompression: disableCompression, + DialTLS: func(netw, addr string, tlsConf *tls.Config) (net.Conn, error) { + return DialTLSWithBackOff(context.Background(), + netw, addr, tlsConf) + }, + TLSClientConfig: tlsConf, + } +} diff --git a/network/transports.go b/network/transports.go index 26ce823957..ff294d8086 100644 --- a/network/transports.go +++ b/network/transports.go @@ -18,6 +18,7 @@ package network import ( "context" + "crypto/tls" "errors" "fmt" "net" @@ -63,11 +64,21 @@ var DialWithBackOff = NewBackoffDialer(backOffTemplate) // between tries. func NewBackoffDialer(backoffConfig wait.Backoff) func(context.Context, string, string) (net.Conn, error) { return func(ctx context.Context, network, address string) (net.Conn, error) { - return dialBackOffHelper(ctx, network, address, backoffConfig, sleepTO) + return dialBackOffHelper(ctx, network, address, backoffConfig, sleepTO, nil) } } -func dialBackOffHelper(ctx context.Context, network, address string, bo wait.Backoff, sleep time.Duration) (net.Conn, error) { +// DialTLSWithBackOff is same with DialWithBackOff but takes tls config. +var DialTLSWithBackOff = NewTLSBackoffDialer(backOffTemplate) + +// NewTLSBackoffDialer is same with NewBackoffDialer but takes tls config. +func NewTLSBackoffDialer(backoffConfig wait.Backoff) func(context.Context, string, string, *tls.Config) (net.Conn, error) { + return func(ctx context.Context, network, address string, tlsConf *tls.Config) (net.Conn, error) { + return dialBackOffHelper(ctx, network, address, backoffConfig, sleepTO, tlsConf) + } +} + +func dialBackOffHelper(ctx context.Context, network, address string, bo wait.Backoff, sleep time.Duration, tlsConf *tls.Config) (net.Conn, error) { dialer := &net.Dialer{ Timeout: bo.Duration, // Initial duration. KeepAlive: 5 * time.Second, @@ -75,7 +86,15 @@ func dialBackOffHelper(ctx context.Context, network, address string, bo wait.Bac } start := time.Now() for { - c, err := dialer.DialContext(ctx, network, address) + var ( + c net.Conn + err error + ) + if tlsConf == nil { + c, err = dialer.DialContext(ctx, network, address) + } else { + c, err = tls.DialWithDialer(dialer, network, address, tlsConf) + } if err != nil { var errNet net.Error if errors.As(err, &errNet) && errNet.Timeout() { @@ -105,6 +124,19 @@ func newHTTPTransport(disableKeepAlives, disableCompression bool, maxIdle, maxId return transport } +func newHTTPSTransport(disableKeepAlives, disableCompression bool, maxIdle, maxIdlePerHost int, tlsConf *tls.Config) http.RoundTripper { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DialContext = DialWithBackOff + transport.DisableKeepAlives = disableKeepAlives + transport.MaxIdleConns = maxIdle + transport.MaxIdleConnsPerHost = maxIdlePerHost + transport.ForceAttemptHTTP2 = false + transport.DisableCompression = disableCompression + + transport.TLSClientConfig = tlsConf + return transport +} + // NewProberTransport creates a RoundTripper that is useful for probing, // since it will not cache connections. func NewProberTransport() http.RoundTripper { @@ -113,6 +145,13 @@ func NewProberTransport() http.RoundTripper { NewH2CTransport()) } +// NewProxyAutoTLSTransport is same with NewProxyAutoTransport but it has tls.Config to create HTTPS request. +func NewProxyAutoTLSTransport(maxIdle, maxIdlePerHost int, tlsConf *tls.Config) http.RoundTripper { + return newAutoTransport( + newHTTPSTransport(false /*disable keep-alives*/, true /*disable auto-compression*/, maxIdle, maxIdlePerHost, tlsConf), + newH2Transport(true /*disable auto-compression*/, tlsConf)) +} + // NewAutoTransport creates a RoundTripper that can use appropriate transport // based on the request's HTTP version. func NewAutoTransport(maxIdle, maxIdlePerHost int) http.RoundTripper { diff --git a/network/transports_test.go b/network/transports_test.go index cdc62baaca..d42277bb98 100644 --- a/network/transports_test.go +++ b/network/transports_test.go @@ -18,6 +18,7 @@ package network import ( "context" + "crypto/tls" "net/http" "net/http/httptest" "strings" @@ -81,7 +82,7 @@ func TestDialWithBackoff(t *testing.T) { bo.Steps = 2 // Timeout. Use special testing IP address. - c, err = dialBackOffHelper(context.Background(), "tcp4", "198.18.0.254:8888", bo, sleepTO) + c, err = dialBackOffHelper(context.Background(), "tcp4", "198.18.0.254:8888", bo, sleepTO, nil) if err == nil { c.Close() t.Error("Unexpected success dialing") @@ -100,3 +101,41 @@ func TestDialWithBackoff(t *testing.T) { } c.Close() } + +func TestDialTLSWithBackoff(t *testing.T) { + tlsConf := &tls.Config{ + InsecureSkipVerify: true, // FIXME: Is it possible to set false for the test server? + ServerName: "example.com", + MinVersion: tls.VersionTLS12, + } + // Nobody's listening on a random port. Usually. + c, err := DialTLSWithBackOff(context.Background(), "tcp4", "127.0.0.1:41482", tlsConf) + if err == nil { + c.Close() + t.Error("Unexpected success dialing") + } + + // Make the test short. + bo := backOffTemplate + bo.Steps = 2 + + // Timeout. Use special testing IP address. + c, err = dialBackOffHelper(context.Background(), "tcp4", "198.18.0.254:8888", bo, sleepTO, tlsConf) + if err == nil { + c.Close() + t.Error("Unexpected success dialing") + } + const expectedErrPrefix = "timed out dialing" + if err == nil || !strings.HasPrefix(err.Error(), expectedErrPrefix) { + t.Errorf("Error = %v, want: %s(...)", err, expectedErrPrefix) + } + + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer s.Close() + + c, err = DialTLSWithBackOff(context.Background(), "tcp4", strings.TrimPrefix(s.URL, "https://"), tlsConf) + if err != nil { + t.Fatal("Dial error =", err) + } + c.Close() +} From 0ffe4207299a131b8efbcbd1937602192a57d493 Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Tue, 5 Apr 2022 22:30:18 +0900 Subject: [PATCH 2/3] Fix lint --- network/transports.go | 8 ++++---- network/transports_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/network/transports.go b/network/transports.go index ff294d8086..d96eda1177 100644 --- a/network/transports.go +++ b/network/transports.go @@ -46,7 +46,7 @@ func newAutoTransport(v1, v2 http.RoundTripper) http.RoundTripper { }) } -const sleepTO = 30 * time.Millisecond +const sleep = 30 * time.Millisecond var backOffTemplate = wait.Backoff{ Duration: 50 * time.Millisecond, @@ -64,7 +64,7 @@ var DialWithBackOff = NewBackoffDialer(backOffTemplate) // between tries. func NewBackoffDialer(backoffConfig wait.Backoff) func(context.Context, string, string) (net.Conn, error) { return func(ctx context.Context, network, address string) (net.Conn, error) { - return dialBackOffHelper(ctx, network, address, backoffConfig, sleepTO, nil) + return dialBackOffHelper(ctx, network, address, backoffConfig, nil) } } @@ -74,11 +74,11 @@ var DialTLSWithBackOff = NewTLSBackoffDialer(backOffTemplate) // NewTLSBackoffDialer is same with NewBackoffDialer but takes tls config. func NewTLSBackoffDialer(backoffConfig wait.Backoff) func(context.Context, string, string, *tls.Config) (net.Conn, error) { return func(ctx context.Context, network, address string, tlsConf *tls.Config) (net.Conn, error) { - return dialBackOffHelper(ctx, network, address, backoffConfig, sleepTO, tlsConf) + return dialBackOffHelper(ctx, network, address, backoffConfig, tlsConf) } } -func dialBackOffHelper(ctx context.Context, network, address string, bo wait.Backoff, sleep time.Duration, tlsConf *tls.Config) (net.Conn, error) { +func dialBackOffHelper(ctx context.Context, network, address string, bo wait.Backoff, tlsConf *tls.Config) (net.Conn, error) { dialer := &net.Dialer{ Timeout: bo.Duration, // Initial duration. KeepAlive: 5 * time.Second, diff --git a/network/transports_test.go b/network/transports_test.go index d42277bb98..7871176ccd 100644 --- a/network/transports_test.go +++ b/network/transports_test.go @@ -82,7 +82,7 @@ func TestDialWithBackoff(t *testing.T) { bo.Steps = 2 // Timeout. Use special testing IP address. - c, err = dialBackOffHelper(context.Background(), "tcp4", "198.18.0.254:8888", bo, sleepTO, nil) + c, err = dialBackOffHelper(context.Background(), "tcp4", "198.18.0.254:8888", bo, nil) if err == nil { c.Close() t.Error("Unexpected success dialing") @@ -120,7 +120,7 @@ func TestDialTLSWithBackoff(t *testing.T) { bo.Steps = 2 // Timeout. Use special testing IP address. - c, err = dialBackOffHelper(context.Background(), "tcp4", "198.18.0.254:8888", bo, sleepTO, tlsConf) + c, err = dialBackOffHelper(context.Background(), "tcp4", "198.18.0.254:8888", bo, tlsConf) if err == nil { c.Close() t.Error("Unexpected success dialing") From 253457ed3c1f9a18bc46c56af3c7f3a2f477771c Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Fri, 8 Apr 2022 23:18:56 +0900 Subject: [PATCH 3/3] Fix review comments --- network/transports_test.go | 67 +++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/network/transports_test.go b/network/transports_test.go index 7871176ccd..7691e16280 100644 --- a/network/transports_test.go +++ b/network/transports_test.go @@ -19,6 +19,8 @@ package network import ( "context" "crypto/tls" + "crypto/x509" + "net" "net/http" "net/http/httptest" "strings" @@ -27,6 +29,11 @@ import ( "k8s.io/apimachinery/pkg/util/sets" ) +const ( + timeoutErr = "timed out dialing" + connectionRefusedErr = "connection refused" +) + func TestHTTPRoundTripper(t *testing.T) { wants := sets.NewString() frt := func(key string) http.RoundTripper { @@ -70,27 +77,17 @@ func TestHTTPRoundTripper(t *testing.T) { } func TestDialWithBackoff(t *testing.T) { - // Nobody's listening on a random port. Usually. - c, err := DialWithBackOff(context.Background(), "tcp4", "127.0.0.1:41482") - if err == nil { - c.Close() - t.Error("Unexpected success dialing") - } - // Make the test short. bo := backOffTemplate bo.Steps = 2 + // Nobody's listening on a random port. Usually. + c, err := dialBackOffHelper(context.Background(), "tcp4", "127.0.0.1:41482", bo, nil) + verifyFailedConnection(t, c, err, connectionRefusedErr) + // Timeout. Use special testing IP address. c, err = dialBackOffHelper(context.Background(), "tcp4", "198.18.0.254:8888", bo, nil) - if err == nil { - c.Close() - t.Error("Unexpected success dialing") - } - const expectedErrPrefix = "timed out dialing" - if err == nil || !strings.HasPrefix(err.Error(), expectedErrPrefix) { - t.Errorf("Error = %v, want: %s(...)", err, expectedErrPrefix) - } + verifyFailedConnection(t, c, err, timeoutErr) s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) defer s.Close() @@ -103,39 +100,43 @@ func TestDialWithBackoff(t *testing.T) { } func TestDialTLSWithBackoff(t *testing.T) { + // Make the test short. + bo := backOffTemplate + bo.Steps = 2 + tlsConf := &tls.Config{ - InsecureSkipVerify: true, // FIXME: Is it possible to set false for the test server? + InsecureSkipVerify: false, ServerName: "example.com", MinVersion: tls.VersionTLS12, } - // Nobody's listening on a random port. Usually. - c, err := DialTLSWithBackOff(context.Background(), "tcp4", "127.0.0.1:41482", tlsConf) - if err == nil { - c.Close() - t.Error("Unexpected success dialing") - } - // Make the test short. - bo := backOffTemplate - bo.Steps = 2 + // Nobody's listening on a random port. Usually. + c, err := dialBackOffHelper(context.Background(), "tcp4", "127.0.0.1:41482", bo, tlsConf) + verifyFailedConnection(t, c, err, connectionRefusedErr) // Timeout. Use special testing IP address. c, err = dialBackOffHelper(context.Background(), "tcp4", "198.18.0.254:8888", bo, tlsConf) - if err == nil { - c.Close() - t.Error("Unexpected success dialing") - } - const expectedErrPrefix = "timed out dialing" - if err == nil || !strings.HasPrefix(err.Error(), expectedErrPrefix) { - t.Errorf("Error = %v, want: %s(...)", err, expectedErrPrefix) - } + verifyFailedConnection(t, c, err, timeoutErr) s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) defer s.Close() + rootCAs := x509.NewCertPool() + rootCAs.AddCert(s.Certificate()) + tlsConf.RootCAs = rootCAs + c, err = DialTLSWithBackOff(context.Background(), "tcp4", strings.TrimPrefix(s.URL, "https://"), tlsConf) if err != nil { t.Fatal("Dial error =", err) } c.Close() } + +func verifyFailedConnection(t *testing.T, c net.Conn, err error, prefix string) { + if err == nil { + c.Close() + t.Error("Unexpected success dialing") + } else if !strings.Contains(err.Error(), prefix) { + t.Errorf("Error = %v, want: %s(...)", err, prefix) + } +}