From abe2f8bac751f40d5af15364e67d510aea1b344a Mon Sep 17 00:00:00 2001 From: rootfs Date: Thu, 6 May 2021 10:57:40 -0400 Subject: [PATCH] init and start kube-apiserver --- pkg/cmd/init.go | 45 ++++++++++------ pkg/controllers/etcd.go | 4 +- pkg/controllers/kube-api.go | 2 +- pkg/util/cert.go | 23 +++++++- pkg/util/config.go | 101 +++++++++++++++++++++++++----------- pkg/util/kubeconfig.go | 6 +-- 6 files changed, 126 insertions(+), 55 deletions(-) diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 933c2ff428..8b745d44c5 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -62,46 +62,57 @@ func initCerts() error { if err != nil { return fmt.Errorf("failed to get host IP: %v", err) } + // store root CA for all + //TODO generate ca bundles for each component + if err := util.StoreRootCA("/etc/kubernetes/ushift-certs/ca-bundle", + "ca-bundle.crt", "ca-bundle.key"); err != nil { + return err + } + // based on https://github.com/openshift/cluster-etcd-operator/blob/master/bindata/bootkube/bootstrap-manifests/etcd-member-pod.yaml#L19 if err := util.GenCerts("/etc/kubernetes/ushift-certs/secrets/etcd-all-serving", "etcd-serving.crt", "etcd-serving.key", []string{"localhost", ip, "127.0.0.1", hostname}); err != nil { return err } - if err := util.StoreRootCA("/etc/kubernetes/ushift-certs/configmaps/etcd-serving-ca", - "ca-bundle.crt", "ca-bundle.key"); err != nil { - return err - } if err := util.GenCerts("/etc/kubernetes/ushift-certs/secrets/etcd-all-peer", "etcd-peer.crt", "etcd-peer.key", []string{"localhost", ip, "127.0.0.1", hostname}); err != nil { return err } - if err := util.StoreRootCA("/etc/kubernetes/ushift-certs/configmaps/etcd-peer-client-ca", - "ca-bundle.crt", "ca-bundle.key"); err != nil { + + // kube-apiserver + if err := util.GenCerts("/etc/kubernetes/ushift-resources/kube-apiserver/secrets/etcd-client", + "tls.crt", "tls.key", + []string{"localhost", ip, "127.0.0.1", hostname}); err != nil { + return err + } + if err := util.GenCerts("/etc/kubernetes/ushift-certs/kube-apiserver/secrets/service-network-serving-certkey", + "tls.crt", "tls.key", + []string{"localhost", ip, "127.0.0.1", hostname}); err != nil { return err } - // kube-apiserver - // etcd-cafile: /etc/kubernetes/ushift-resources/configmaps/etcd-serving-ca/ca-bundle.crt - if err := util.StoreRootCA("/etc/kubernetes/ushift-resources/configmaps/etcd-serving-ca", - "ca-bundle.crt", "ca-bundle.key"); err != nil { + if err := util.GenKeys("/etc/kubernetes/ushift-resources/kube-apiserver/secrets/service-account-signing-key", + "service-account.crt", "service-account.key"); err != nil { return err } - // etcd-certfile: /etc/kubernetes/ushift-resources/secrets/etcd-client/tls.crt - // etcd-keyfile: /etc/kubernetes/ushift-resources/secrets/etcd-client/tls.key - if err := util.GenCerts("/etc/kubernetes/ushift-resources/secrets/etcd-client", + if err := util.GenCerts("/etc/kubernetes/ushift-certs/kube-apiserver/secrets/aggregator-client", "tls.crt", "tls.key", []string{"localhost", ip, "127.0.0.1", hostname}); err != nil { return err } - // kube-apiserver - // client-ca-file: /etc/kubernetes/ushift-certs/configmaps/client-ca/ca-bundle.crt - if err := util.StoreRootCA("/etc/kubernetes/ushift-certs/configmaps/client-ca/", - "ca-bundle.crt", "ca-bundle.key"); err != nil { + if err := util.GenCerts("/etc/kubernetes/ushift-resources/kube-apiserver/secrets/kubelet-client", + "tls.crt", "tls.key", + []string{"localhost", ip, "127.0.0.1", hostname}); err != nil { return err } + if err := util.GenKeys("/etc/kubernetes/ushift-resources/kube-apiserver/sa-public-key", + "serving-ca.pub", "serving-ca.key"); err != nil { + return err + } + /* // kubelet // kubelet-certificate-authority: /etc/kubernetes/ushift-resources/configmaps/kubelet-serving-ca/ca-bundle.crt diff --git a/pkg/controllers/etcd.go b/pkg/controllers/etcd.go index e067d9521c..61b02d5b3c 100644 --- a/pkg/controllers/etcd.go +++ b/pkg/controllers/etcd.go @@ -66,13 +66,13 @@ func StartEtcd(ready chan bool) error { cfg.CipherSuites = tlsCipherSuites cfg.ClientTLSInfo.CertFile = "/etc/kubernetes/ushift-certs/secrets/etcd-all-serving/etcd-serving.crt" cfg.ClientTLSInfo.KeyFile = "/etc/kubernetes/ushift-certs/secrets/etcd-all-serving/etcd-serving.key" - cfg.ClientTLSInfo.TrustedCAFile = "/etc/kubernetes/ushift-certs/configmaps/etcd-serving-ca/ca-bundle.crt" + cfg.ClientTLSInfo.TrustedCAFile = "/etc/kubernetes/ushift-certs/ca-bundle/ca-bundle.crt" cfg.ClientTLSInfo.ClientCertAuth = false cfg.ClientTLSInfo.InsecureSkipVerify = true //TODO after fix GenCert to generate client cert cfg.PeerTLSInfo.CertFile = "/etc/kubernetes/ushift-certs/secrets/etcd-all-peer/etcd-peer.crt" cfg.PeerTLSInfo.KeyFile = "/etc/kubernetes/ushift-certs/secrets/etcd-all-peer/etcd-peer.key" - cfg.PeerTLSInfo.TrustedCAFile = "/etc/kubernetes/ushift-certs/configmaps/etcd-peer-client-ca/ca-bundle.crt" + cfg.PeerTLSInfo.TrustedCAFile = "/etc/kubernetes/ushift-certs/ca-bundle/ca-bundle.crt" cfg.PeerTLSInfo.ClientCertAuth = false cfg.PeerTLSInfo.InsecureSkipVerify = true //TODO after fix GenCert to generate client cert diff --git a/pkg/controllers/kube-api.go b/pkg/controllers/kube-api.go index 5af5245027..485a0dc977 100644 --- a/pkg/controllers/kube-api.go +++ b/pkg/controllers/kube-api.go @@ -40,7 +40,7 @@ func KubeAPIServer(args []string, ready chan bool) error { apiArgs := []string{ "--openshift-config=/etc/kubernetes/ushift-resources/kube-apiserver/config/config.yaml", "--advertise-address=" + ip, - "-v=3", + //"-v=3", } if err := command.ParseFlags(apiArgs); err != nil { return err diff --git a/pkg/util/cert.go b/pkg/util/cert.go index aaddb711f8..430053b4f0 100644 --- a/pkg/util/cert.go +++ b/pkg/util/cert.go @@ -89,7 +89,7 @@ func GenCerts(dir, certFilename, keyFilename string, svcName []string) error { IPAddresses: ip, Validity: defaultDuration, IsCA: false, - KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, /*| x509.KeyUsageCertSign*/ + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, Subject: pkix.Name{CommonName: dns[0], OrganizationalUnit: []string{defaultOrganizationalUnit}}, } @@ -107,6 +107,27 @@ func GenCerts(dir, certFilename, keyFilename string, svcName []string) error { return err } +// GenKeys generates and save rsa keys +func GenKeys(dir, pubFilename, keyFilename string) error { + key, err := PrivateKey() + if err != nil { + return err + } + pub := &key.PublicKey + pubBuff, err := PublicKeyToPem(pub) + if err != nil { + return err + } + keyBuff := PrivateKeyToPem(key) + os.MkdirAll(dir, 0700) + pubPath := filepath.Join(dir, pubFilename) + keyPath := filepath.Join(dir, keyFilename) + ioutil.WriteFile(pubPath, pubBuff, 0644) + ioutil.WriteFile(keyPath, keyBuff, 0644) + return err + +} + // based on github.com/hypershift/certs/tls.go // CertCfg contains all needed fields to configure a new certificate diff --git a/pkg/util/config.go b/pkg/util/config.go index 0547ee49a0..c15c71a60a 100644 --- a/pkg/util/config.go +++ b/pkg/util/config.go @@ -1,6 +1,7 @@ package util import ( + "io/ioutil" "os" "path/filepath" "strconv" @@ -11,12 +12,52 @@ const ( port = 32444 ) +func kubeAPIAuditPolicyFile(path string) error { + data := []byte(` +apiVersion: audit.k8s.io/v1beta1 +kind: Policy +metadata: + name: Default +# Don't generate audit events for all requests in RequestReceived stage. +omitStages: +- "RequestReceived" +rules: +# Don't log requests for events +- level: None + resources: + - group: "" + resources: ["events"] +# Don't log oauth tokens as metadata.name is the secret +- level: None + resources: + - group: "oauth.openshift.io" + resources: ["oauthaccesstokens", "oauthauthorizetokens"] +# Don't log authenticated requests to certain non-resource URL paths. +- level: None + userGroups: ["system:authenticated", "system:unauthenticated"] + nonResourceURLs: + - "/api*" # Wildcard matching. + - "/version" + - "/healthz" + - "/readyz" +# A catch-all rule to log all other requests at the Metadata level. +- level: Metadata + # Long-running requests like watches that fall under this rule will not + # generate an audit event in RequestReceived. + omitStages: + - "RequestReceived"`) + os.MkdirAll(filepath.Dir(path), os.FileMode(0755)) + return ioutil.WriteFile(path, data, 0644) +} + // KubeAPIServerConfig creates a config for kube-apiserver to use in --openshift-config option func KubeAPIServerConfig(path, svcCIDR string) error { // based on https://github.com/openshift/cluster-kube-apiserver-operator/blob/master/bindata/v4.1.0/config/defaultconfig.yaml configTemplate := template.Must(template.New("config").Parse(` apiVersion: kubecontrolplane.config.openshift.io/v1 -kind: KubeAPIServerConfig +kind: KubeAPIServerConfig +serviceAccountPublicKeyFiles: + - /etc/kubernetes/ushift-resources/kube-apiserver/sa-public-key/serving-ca.pub admission: pluginConfig: network.openshift.io/ExternalIPRanger: @@ -45,9 +86,7 @@ apiServerArguments: audit-log-path: - /var/log/kube-apiserver/audit.log audit-policy-file: - - /etc/kubernetes/ushift-resources/configmaps/kube-apiserver-audit-policies/default.yaml - client-ca-file: - - /etc/kubernetes/ushift-certs/configmaps/client-ca/ca-bundle.crt + - /etc/kubernetes/ushift-resources/kube-apiserver-audit-policies/default.yaml enable-admission-plugins: - CertificateApproval - CertificateSigning @@ -66,7 +105,6 @@ apiServerArguments: - PodTolerationRestriction - Priority - ResourceQuota - - Class - ServiceAccount - StorageObjectInUseProtection - TaintNodesByCondition @@ -102,14 +140,6 @@ apiServerArguments: - "true" endpoint-reconciler-type: - "lease" - etcd-cafile: - - /etc/kubernetes/ushift-resources/configmaps/etcd-serving-ca/ca-bundle.crt - etcd-certfile: - - /etc/kubernetes/ushift-resources/secrets/etcd-client/tls.crt - etcd-keyfile: - - /etc/kubernetes/ushift-resources/secrets/etcd-client/tls.key - etcd-prefix: - - kubernetes.io event-ttl: - 3h goaway-chance: @@ -119,11 +149,11 @@ apiServerArguments: insecure-port: - "0" kubelet-certificate-authority: - - /etc/kubernetes/ushift-resources/configmaps/kubelet-serving-ca/ca-bundle.crt + - /etc/kubernetes/ushift-certs/ca-bundle/ca-bundle.crt kubelet-client-certificate: - - /etc/kubernetes/ushift-resources/secrets/kubelet-client/tls.crt + - /etc/kubernetes/ushift-resources/kube-apiserver/secrets/kubelet-client/tls.crt kubelet-client-key: - - /etc/kubernetes/ushift-resources/secrets/kubelet-client/tls.key + - /etc/kubernetes/ushift-resources/kube-apiserver/secrets/kubelet-client/tls.key kubelet-https: - "true" kubelet-preferred-address-types: @@ -141,15 +171,15 @@ apiServerArguments: min-request-timeout: - "3600" proxy-client-cert-file: - - /etc/kubernetes/ushift-certs/secrets/aggregator-client/tls.crt + - /etc/kubernetes/ushift-certs/kube-apiserver/secrets/aggregator-client/tls.crt proxy-client-key-file: - - /etc/kubernetes/ushift-certs/secrets/aggregator-client/tls.key + - /etc/kubernetes/ushift-certs/kube-apiserver/secrets/aggregator-client/tls.key requestheader-allowed-names: - kube-apiserver-proxy - system:kube-apiserver-proxy - system:openshift-aggregator requestheader-client-ca-file: - - /etc/kubernetes/ushift-certs/configmaps/aggregator-client-ca/ca-bundle.crt + - /etc/kubernetes/ushift-certs/ca-bundle/ca-bundle.crt requestheader-extra-headers-prefix: - X-Remote-Extra- requestheader-group-headers: @@ -168,9 +198,23 @@ apiServerArguments: storage-media-type: - application/vnd.kubernetes.protobuf tls-cert-file: - - /etc/kubernetes/ushift-certs/secrets/service-network-serving-certkey/tls.crt + - /etc/kubernetes/ushift-certs/kube-apiserver/secrets/service-network-serving-certkey/tls.crt tls-private-key-file: - - /etc/kubernetes/ushift-certs/secrets/service-network-serving-certkey/tls.key + - /etc/kubernetes/ushift-certs/kube-apiserver/secrets/service-network-serving-certkey/tls.key + service-account-issuer: + - https://kubernetes.default.svc + service-account-signing-key-file: + - /etc/kubernetes/ushift-resources/kube-apiserver/secrets/service-account-signing-key/service-account.key + etcd-cafile: + - /etc/kubernetes/ushift-certs/ca-bundle/ca-bundle.crt + etcd-certfile: + - /etc/kubernetes/ushift-resources/kube-apiserver/secrets/etcd-client/tls.crt + etcd-keyfile: + - /etc/kubernetes/ushift-resources/kube-apiserver/secrets/etcd-client/tls.key + etcd-prefix: + - kubernetes.io + etcd-servers: + - https://127.0.0.1:2379 authConfig: oauthMetadataFile: "" consolePublicURL: "" @@ -181,6 +225,10 @@ servingInfo: bindAddress: 0.0.0.0:6443 # set by observe_network.go bindNetwork: tcp4 # set by observe_network.go namedCertificates: null # set by observe_apiserver.go`)) + + if err := kubeAPIAuditPolicyFile("/etc/kubernetes/ushift-resources/kube-apiserver-audit-policies/default.yaml"); err != nil { + return err + } data := struct { ServiceCIDR string }{ @@ -236,18 +284,11 @@ extendedArguments: - "0" cert-dir: - "/var/run/kubernetes" - root-ca-file: - - "/etc/kubernetes/ushift-resources/configmaps/serviceaccount-ca/ca-bundle.crt" - service-account-private-key-file: - - "/etc/kubernetes/ushift-resources/secrets/service-account-private-key/service-account.key" - cluster-signing-cert-file: - - "/etc/kubernetes/ushift-certs/secrets/csr-signer/tls.crt" - cluster-signing-key-file: - - "/etc/kubernetes/ushift-certs/secrets/csr-signer/tls.key" kube-api-qps: - "150" # this is a historical values kube-api-burst: - - "300" # this is a historical values`)) + - "300" # this is a historical values + `)) data := struct { ClientCACert, KubeConfig, ServingCert, ServingKey, ServingClientCert, IngressDomain, EtcdUrl, EtcdCert, EtcdKey, EtcdCA string diff --git a/pkg/util/kubeconfig.go b/pkg/util/kubeconfig.go index edbeeed197..2d438ee977 100644 --- a/pkg/util/kubeconfig.go +++ b/pkg/util/kubeconfig.go @@ -7,7 +7,7 @@ import ( ) // Kubeconfig creates a kubeconfig -func Kubeconfig(dir, filename, endpoint string) error { +func Kubeconfig(path, endpoint string) error { kubeconfigTemplate := template.Must(template.New("kubeconfig").Parse(` apiVersion: v1 kind: Config @@ -45,9 +45,7 @@ users: ClientCert: clientCert, ClientKey: clientKey, } - - os.MkdirAll(dir, 0700) - path := filepath.Join(dir, filename) + os.MkdirAll(filepath.Dir(path), os.FileMode(0755)) output, err := os.Create(path) if err != nil {