From c36653e0db77c32aced45dd3c87382911836fa23 Mon Sep 17 00:00:00 2001 From: Tomas Nozicka Date: Mon, 4 May 2020 14:36:15 +0200 Subject: [PATCH] UPSTREAM: 90360: Fix client-ca dynamic reload in apiserver --- .../cmd/kube-apiserver/app/server.go | 41 +--- .../k8s.io/kubernetes/cmd/kubelet/app/auth.go | 33 ++- .../kubernetes/cmd/kubelet/app/server.go | 3 +- .../kubeapiserver/options/authentication.go | 70 ++++-- .../apiserver/certreload/certreload_test.go | 229 +++++++++++++++--- 5 files changed, 265 insertions(+), 111 deletions(-) diff --git a/vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go b/vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go index 968623e1650e..6ad81e8f3aee 100644 --- a/vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go +++ b/vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go @@ -34,7 +34,6 @@ import ( "k8s.io/kubernetes/openshift-kube-apiserver/enablement" "k8s.io/kubernetes/openshift-kube-apiserver/openshiftkubeapiserver" - "github.com/go-openapi/spec" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" @@ -45,7 +44,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" utilwait "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authorization/authorizer" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" "k8s.io/apiserver/pkg/endpoints/request" @@ -79,7 +77,6 @@ import ( "k8s.io/kubernetes/pkg/apis/core" v1 "k8s.io/kubernetes/pkg/apis/core/v1" "k8s.io/kubernetes/pkg/capabilities" - serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" "k8s.io/kubernetes/pkg/features" generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/kubeapiserver" @@ -96,7 +93,6 @@ import ( rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" "k8s.io/kubernetes/pkg/serviceaccount" utilflag "k8s.io/kubernetes/pkg/util/flag" - "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" ) const ( @@ -489,9 +485,6 @@ func buildGenericConfig( if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil { return } - if lastErr = s.Authentication.ApplyTo(genericConfig); lastErr != nil { - return - } if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil { return } @@ -549,9 +542,8 @@ func buildGenericConfig( } versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) - genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, genericConfig.EgressSelector, clientgoExternalClient, versionedInformers) - if err != nil { - lastErr = fmt.Errorf("invalid authentication config: %v", err) + // Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present + if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers); lastErr != nil { return } @@ -618,35 +610,6 @@ func buildGenericConfig( return } -// BuildAuthenticator constructs the authenticator -func BuildAuthenticator(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) { - authenticatorConfig, err := s.Authentication.ToAuthenticationConfig() - if err != nil { - return nil, nil, err - } - if s.Authentication.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { - authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient( - extclient, - versionedInformer.Core().V1().Secrets().Lister(), - versionedInformer.Core().V1().ServiceAccounts().Lister(), - versionedInformer.Core().V1().Pods().Lister(), - ) - } - authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator( - versionedInformer.Core().V1().Secrets().Lister().Secrets(metav1.NamespaceSystem), - ) - - if EgressSelector != nil { - egressDialer, err := EgressSelector.Lookup(egressselector.Master.AsNetworkContext()) - if err != nil { - return nil, nil, err - } - authenticatorConfig.CustomDial = egressDialer - } - - return authenticatorConfig.New() -} - // BuildAuthorizer constructs the authorizer func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) { authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers) diff --git a/vendor/k8s.io/kubernetes/cmd/kubelet/app/auth.go b/vendor/k8s.io/kubernetes/cmd/kubelet/app/auth.go index 22a0285d8b95..6eadf29bb1b1 100644 --- a/vendor/k8s.io/kubernetes/cmd/kubelet/app/auth.go +++ b/vendor/k8s.io/kubernetes/cmd/kubelet/app/auth.go @@ -36,7 +36,8 @@ import ( ) // BuildAuth creates an authenticator, an authorizer, and a matching authorizer attributes getter compatible with the kubelet's needs -func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, error) { +// It returns AuthInterface, a run method to start internal controllers (like cert reloading) and error. +func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, func(<-chan struct{}), error) { // Get clients, if provided var ( tokenClient authenticationclient.TokenReviewInterface @@ -47,47 +48,55 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel sarClient = client.AuthorizationV1().SubjectAccessReviews() } - authenticator, err := BuildAuthn(tokenClient, config.Authentication) + authenticator, runAuthenticatorCAReload, err := BuildAuthn(tokenClient, config.Authentication) if err != nil { - return nil, err + return nil, nil, err } attributes := server.NewNodeAuthorizerAttributesGetter(nodeName) authorizer, err := BuildAuthz(sarClient, config.Authorization) if err != nil { - return nil, err + return nil, nil, err } - return server.NewKubeletAuth(authenticator, attributes, authorizer), nil + return server.NewKubeletAuth(authenticator, attributes, authorizer), runAuthenticatorCAReload, nil } // BuildAuthn creates an authenticator compatible with the kubelet's needs -func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, error) { - var clientCertificateCAContentProvider authenticatorfactory.CAContentProvider +func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, func(<-chan struct{}), error) { + var dynamicCAContentFromFile *dynamiccertificates.DynamicFileCAContent var err error if len(authn.X509.ClientCAFile) > 0 { - clientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile) + dynamicCAContentFromFile, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile) if err != nil { - return nil, err + return nil, nil, err } } authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{ Anonymous: authn.Anonymous.Enabled, CacheTTL: authn.Webhook.CacheTTL.Duration, - ClientCertificateCAContentProvider: clientCertificateCAContentProvider, + ClientCertificateCAContentProvider: dynamicCAContentFromFile, } if authn.Webhook.Enabled { if client == nil { - return nil, errors.New("no client provided, cannot use webhook authentication") + return nil, nil, errors.New("no client provided, cannot use webhook authentication") } authenticatorConfig.TokenAccessReviewClient = client } authenticator, _, err := authenticatorConfig.New() - return authenticator, err + if err != nil { + return nil, nil, err + } + + return authenticator, func(stopCh <-chan struct{}) { + if dynamicCAContentFromFile != nil { + go dynamicCAContentFromFile.Run(1, stopCh) + } + }, err } // BuildAuthz creates an authorizer compatible with the kubelet's needs diff --git a/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go b/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go index 9f772a143476..df059e385b1d 100644 --- a/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go +++ b/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go @@ -608,11 +608,12 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate f } if kubeDeps.Auth == nil { - auth, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration) + auth, runAuthenticatorCAReload, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration) if err != nil { return err } kubeDeps.Auth = auth + runAuthenticatorCAReload(stopCh) } var cgroupRoots []string diff --git a/vendor/k8s.io/kubernetes/pkg/kubeapiserver/options/authentication.go b/vendor/k8s.io/kubernetes/pkg/kubeapiserver/options/authentication.go index c7b4ac5bdb56..1c5a02dbb649 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubeapiserver/options/authentication.go +++ b/vendor/k8s.io/kubernetes/pkg/kubeapiserver/options/authentication.go @@ -24,17 +24,24 @@ import ( "time" "github.com/spf13/pflag" - "k8s.io/klog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/authenticator" genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/egressselector" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog" + openapicommon "k8s.io/kube-openapi/pkg/common" + serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" "k8s.io/kubernetes/pkg/features" kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" ) type BuiltInAuthenticationOptions struct { @@ -421,37 +428,62 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat return ret, nil } -func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error { +// ApplyTo requires already applied OpenAPIConfig and EgressSelector if present. +func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.AuthenticationInfo, secureServing *genericapiserver.SecureServingInfo, egressSelector *egressselector.EgressSelector, openAPIConfig *openapicommon.Config, extclient kubernetes.Interface, versionedInformer informers.SharedInformerFactory) error { if o == nil { return nil } - if o.ClientCert != nil { - clientCertificateCAContentProvider, err := o.ClientCert.GetClientCAContentProvider() - if err != nil { + if openAPIConfig == nil { + return errors.New("uninitialized OpenAPIConfig") + } + + authenticatorConfig, err := o.ToAuthenticationConfig() + if err != nil { + return err + } + + if authenticatorConfig.ClientCAContentProvider != nil { + if err = authInfo.ApplyClientCert(authenticatorConfig.ClientCAContentProvider, secureServing); err != nil { return fmt.Errorf("unable to load client CA file: %v", err) } - if err = c.Authentication.ApplyClientCert(clientCertificateCAContentProvider, c.SecureServing); err != nil { + } + if authenticatorConfig.RequestHeaderConfig != nil && authenticatorConfig.RequestHeaderConfig.CAContentProvider != nil { + if err = authInfo.ApplyClientCert(authenticatorConfig.RequestHeaderConfig.CAContentProvider, secureServing); err != nil { return fmt.Errorf("unable to load client CA file: %v", err) } } - if o.RequestHeader != nil { - requestHeaderConfig, err := o.RequestHeader.ToAuthenticationRequestHeaderConfig() + + authInfo.SupportsBasicAuth = o.PasswordFile != nil && len(o.PasswordFile.BasicAuthFile) > 0 + + authInfo.APIAudiences = o.APIAudiences + if o.ServiceAccounts != nil && o.ServiceAccounts.Issuer != "" && len(o.APIAudiences) == 0 { + authInfo.APIAudiences = authenticator.Audiences{o.ServiceAccounts.Issuer} + } + + if o.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { + authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient( + extclient, + versionedInformer.Core().V1().Secrets().Lister(), + versionedInformer.Core().V1().ServiceAccounts().Lister(), + versionedInformer.Core().V1().Pods().Lister(), + ) + } + authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator( + versionedInformer.Core().V1().Secrets().Lister().Secrets(metav1.NamespaceSystem), + ) + + if egressSelector != nil { + egressDialer, err := egressSelector.Lookup(egressselector.Master.AsNetworkContext()) if err != nil { - return fmt.Errorf("unable to create request header authentication config: %v", err) - } - if requestHeaderConfig != nil { - if err = c.Authentication.ApplyClientCert(requestHeaderConfig.CAContentProvider, c.SecureServing); err != nil { - return fmt.Errorf("unable to load client CA file: %v", err) - } + return err } + authenticatorConfig.CustomDial = egressDialer } - c.Authentication.SupportsBasicAuth = o.PasswordFile != nil && len(o.PasswordFile.BasicAuthFile) > 0 - - c.Authentication.APIAudiences = o.APIAudiences - if o.ServiceAccounts != nil && o.ServiceAccounts.Issuer != "" && len(o.APIAudiences) == 0 { - c.Authentication.APIAudiences = authenticator.Audiences{o.ServiceAccounts.Issuer} + authInfo.Authenticator, openAPIConfig.SecurityDefinitions, err = authenticatorConfig.New() + if err != nil { + return err } return nil diff --git a/vendor/k8s.io/kubernetes/test/integration/apiserver/certreload/certreload_test.go b/vendor/k8s.io/kubernetes/test/integration/apiserver/certreload/certreload_test.go index f35d263203f1..e12bb9e6ce63 100644 --- a/vendor/k8s.io/kubernetes/test/integration/apiserver/certreload/certreload_test.go +++ b/vendor/k8s.io/kubernetes/test/integration/apiserver/certreload/certreload_test.go @@ -19,63 +19,150 @@ package podlogs import ( "bytes" "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "fmt" "io/ioutil" + "math/big" "path" "strings" "testing" "time" + authorizationv1 "k8s.io/api/authorization/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/dynamiccertificates" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" "k8s.io/client-go/util/cert" "k8s.io/component-base/cli/flag" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/integration/framework" ) +type caWithClient struct { + CACert []byte + ClientCert []byte + ClientKey []byte +} + +func newTestCAWithClient(caSubject pkix.Name, caSerial *big.Int, clientSubject pkix.Name, subjectSerial *big.Int) (*caWithClient, error) { + ca := &x509.Certificate{ + SerialNumber: caSerial, + Subject: caSubject, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, err + } + + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivateKey.PublicKey, caPrivateKey) + if err != nil { + return nil, err + } + + caPEM := new(bytes.Buffer) + err = pem.Encode(caPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + if err != nil { + return nil, err + } + + clientCert := &x509.Certificate{ + SerialNumber: subjectSerial, + Subject: clientSubject, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + + clientCertPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, err + } + + clientCertPrivateKeyPEM := new(bytes.Buffer) + err = pem.Encode(clientCertPrivateKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(clientCertPrivateKey), + }) + if err != nil { + return nil, err + } + + clientCertBytes, err := x509.CreateCertificate(rand.Reader, clientCert, ca, &clientCertPrivateKey.PublicKey, caPrivateKey) + if err != nil { + return nil, err + } + + clientCertPEM := new(bytes.Buffer) + err = pem.Encode(clientCertPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: clientCertBytes, + }) + if err != nil { + return nil, err + } + + return &caWithClient{ + CACert: caPEM.Bytes(), + ClientCert: clientCertPEM.Bytes(), + ClientKey: clientCertPrivateKeyPEM.Bytes(), + }, nil +} + func TestClientCA(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) - // I have no idea what this cert is, but it doesn't matter, we just want something that always fails validation - differentClientCA := []byte(`-----BEGIN CERTIFICATE----- -MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV -BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX -DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo -b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa -dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0 -r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD -XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp -7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E -j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P -BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg -hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD -ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6 -ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc -T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF -bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3 -M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0 -YkNtGc1RUDHwecCTFpJtPb7Yu/E= ------END CERTIFICATE----- -`) - differentFrontProxyCA := []byte(`-----BEGIN CERTIFICATE----- -MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw -GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx -MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB -BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb -KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC -BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU -K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q -a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5 -MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= ------END CERTIFICATE----- + frontProxyCA, err := newTestCAWithClient( + pkix.Name{ + CommonName: "test-front-proxy-ca", + }, + big.NewInt(43), + pkix.Name{ + CommonName: "test-aggregated-apiserver", + Organization: []string{"system:masters"}, + }, + big.NewInt(86), + ) + if err != nil { + t.Error(err) + return + } + + clientCA, err := newTestCAWithClient( + pkix.Name{ + CommonName: "test-client-ca", + }, + big.NewInt(42), + pkix.Name{ + CommonName: "system:admin", + Organization: []string{"system:masters"}, + }, + big.NewInt(84), + ) + if err != nil { + t.Error(err) + return + } -`) clientCAFilename := "" frontProxyCAFilename := "" @@ -84,12 +171,13 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024 clientCAFilename = opts.Authentication.ClientCert.ClientCA frontProxyCAFilename = opts.Authentication.RequestHeader.ClientCAFile + opts.Authentication.RequestHeader.AllowedNames = append(opts.Authentication.RequestHeader.AllowedNames, "test-aggregated-apiserver") dynamiccertificates.FileRefreshDuration = 1 * time.Second }, }) // wait for request header info - err := wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 1)) + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 1)) if err != nil { t.Fatal(err) } @@ -100,10 +188,10 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= } // when we run this the second time, we know which one we are expecting - if err := ioutil.WriteFile(clientCAFilename, differentClientCA, 0644); err != nil { + if err := ioutil.WriteFile(clientCAFilename, clientCA.CACert, 0644); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(frontProxyCAFilename, differentFrontProxyCA, 0644); err != nil { + if err := ioutil.WriteFile(frontProxyCAFilename, frontProxyCA.CACert, 0644); err != nil { t.Fatal(err) } @@ -114,7 +202,7 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= t.Fatal(err) } - expectedCAs := []string{"webhook-test.default.svc", "My Client"} + expectedCAs := []string{"test-client-ca", "test-front-proxy-ca"} if len(expectedCAs) != len(acceptableCAs) { t.Fatal(strings.Join(acceptableCAs, ":")) } @@ -129,7 +217,7 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= if err != nil { t.Error(err) } - err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=", 1)) + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", string(frontProxyCA.CACert), 1)) if err != nil { t.Error(err) } @@ -138,7 +226,68 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= if err != nil { t.Error(err) } - err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", "M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0", 1)) + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", string(clientCA.CACert), 1)) + if err != nil { + t.Error(err) + } + + // Test an aggregated apiserver client (signed by the new front proxy CA) is authorized + extensionApiserverClient, err := kubernetes.NewForConfig(&rest.Config{ + Host: kubeconfig.Host, + TLSClientConfig: rest.TLSClientConfig{ + CAData: kubeconfig.TLSClientConfig.CAData, + CAFile: kubeconfig.TLSClientConfig.CAFile, + ServerName: kubeconfig.TLSClientConfig.ServerName, + KeyData: frontProxyCA.ClientKey, + CertData: frontProxyCA.ClientCert, + }, + }) + if err != nil { + t.Error(err) + return + } + + // Call an endpoint to make sure we are authenticated + err = extensionApiserverClient.AuthorizationV1().RESTClient(). + Post(). + Resource("subjectaccessreviews"). + VersionedParams(&metav1.CreateOptions{}, scheme.ParameterCodec). + Body(&authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Verb: "create", + Resource: "pods", + Namespace: "default", + }, + User: "deads2k", + }, + }). + SetHeader("X-Remote-User", "test-aggregated-apiserver"). + SetHeader("X-Remote-Group", "system:masters"). + Do(context.Background()). + Into(&authorizationv1.SubjectAccessReview{}) + if err != nil { + t.Error(err) + } + + // Test a client signed by the new ClientCA is authorized + testClient, err := kubernetes.NewForConfig(&rest.Config{ + Host: kubeconfig.Host, + TLSClientConfig: rest.TLSClientConfig{ + CAData: kubeconfig.TLSClientConfig.CAData, + CAFile: kubeconfig.TLSClientConfig.CAFile, + ServerName: kubeconfig.TLSClientConfig.ServerName, + KeyData: clientCA.ClientKey, + CertData: clientCA.ClientCert, + }, + }) + if err != nil { + t.Error(err) + return + } + + // Call an endpoint to make sure we are authenticated + _, err = testClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) if err != nil { t.Error(err) }