From 426ba2d68666882330782b4b85fb74ba9e31bd71 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 16:23:57 -0500 Subject: [PATCH 01/17] USHIFT-936: align the DNS portion of the config data structures --- .../openshift/microshift/pkg/config/config.go | 11 ++++++----- pkg/cmd/init.go | 8 ++++---- pkg/cmd/run.go | 2 +- pkg/cmd/showConfig.go | 4 +--- pkg/components/render.go | 2 +- pkg/config/config.go | 11 ++++++----- pkg/config/config_test.go | 8 ++++++-- pkg/controllers/kube-apiserver.go | 2 +- 8 files changed, 26 insertions(+), 22 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 97f9847e1c..e905540e03 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -87,11 +87,12 @@ type MicroshiftConfig struct { // Determines if kube-apiserver controller should configure the // KASAdvertiseAddress in the loopback interface. Automatically computed. SkipKASInterface bool `json:"-"` - BaseDomain string `json:"baseDomain"` Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` + + DNS DNS `json:"-"` } // Top level config file @@ -249,7 +250,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { SubjectAltNames: subjectAltNames, NodeName: strings.ToLower(nodeName), NodeIP: nodeIP, - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.42.0.0/16", @@ -411,9 +414,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { if config.Network.ServiceNodePortRange != "" { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } - if config.DNS.BaseDomain != "" { - c.BaseDomain = config.DNS.BaseDomain - } + c.DNS = config.DNS if len(config.ApiServer.SubjectAltNames) > 0 { c.SubjectAltNames = config.ApiServer.SubjectAltNames } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index b3ab6c026e..688dc8fcca 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -72,7 +72,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err externalCertNames := []string{ cfg.NodeName, - "api." + cfg.BaseDomain, + "api." + cfg.DNS.BaseDomain, } externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) // When Kube apiserver advertise address matches the node IP we can not add @@ -229,7 +229,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, Hostnames: []string{ - "*.apps." + cfg.BaseDomain, // wildcard for any additional auto-generated domains + "*.apps." + cfg.DNS.BaseDomain, // wildcard for any additional auto-generated domains }, }, ), @@ -285,8 +285,8 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - "api." + cfg.BaseDomain, - "api-int." + cfg.BaseDomain, + "api." + cfg.DNS.BaseDomain, + "api-int." + cfg.DNS.BaseDomain, apiServerServiceIP.String(), }, }, diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index e8c3558862..ccc6c7fc94 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -62,7 +62,7 @@ func RunMicroshift(cfg *config.MicroshiftConfig) error { cfg.Cluster.ServiceCIDR, ".svc", ".cluster.local", - "."+cfg.BaseDomain); err != nil { + "."+cfg.DNS.BaseDomain); err != nil { klog.Fatal(err) } diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 21c57f5f08..1adfd5a7b0 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -55,9 +55,7 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, }, - DNS: config.DNS{ - BaseDomain: cfg.BaseDomain, - }, + DNS: cfg.DNS, Node: config.Node{ HostnameOverride: cfg.NodeName, NodeIP: cfg.NodeIP, diff --git a/pkg/components/render.go b/pkg/components/render.go index b24723c5db..d826db28eb 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -28,7 +28,7 @@ func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderPar "ClusterCIDR": cfg.Cluster.ClusterCIDR, "ServiceCIDR": cfg.Cluster.ServiceCIDR, "ClusterDNS": cfg.Cluster.DNS, - "BaseDomain": cfg.BaseDomain, + "BaseDomain": cfg.DNS.BaseDomain, } for k, v := range extra { params[k] = v diff --git a/pkg/config/config.go b/pkg/config/config.go index 97f9847e1c..e905540e03 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -87,11 +87,12 @@ type MicroshiftConfig struct { // Determines if kube-apiserver controller should configure the // KASAdvertiseAddress in the loopback interface. Automatically computed. SkipKASInterface bool `json:"-"` - BaseDomain string `json:"baseDomain"` Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` + + DNS DNS `json:"-"` } // Top level config file @@ -249,7 +250,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { SubjectAltNames: subjectAltNames, NodeName: strings.ToLower(nodeName), NodeIP: nodeIP, - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.42.0.0/16", @@ -411,9 +414,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { if config.Network.ServiceNodePortRange != "" { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } - if config.DNS.BaseDomain != "" { - c.BaseDomain = config.DNS.BaseDomain - } + c.DNS = config.DNS if len(config.ApiServer.SubjectAltNames) > 0 { c.SubjectAltNames = config.ApiServer.SubjectAltNames } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 3616c79a28..78401276dd 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -64,7 +64,9 @@ func TestConfigFile(t *testing.T) { NodeName: "node1", NodeIP: "1.2.3.4", KASAdvertiseAddress: "6.7.8.9", - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.20.30.40/16", @@ -159,7 +161,9 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { NodeIP: "1.2.3.4", KASAdvertiseAddress: "6.7.8.9", SkipKASInterface: true, - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.20.30.40/16", diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index 49c0edbae4..a1801332b5 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -172,7 +172,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { APIVersion: "route.openshift.io/v1", Kind: "HostAssignmentAdmissionConfig", }, - Domain: "apps." + cfg.BaseDomain, + Domain: "apps." + cfg.DNS.BaseDomain, }, }, }, From be16c7dd21318ef91f8c8e865d63bc41c96c7c80 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 16:40:43 -0500 Subject: [PATCH 02/17] USHIFT-936: align the Node portion of the config data structures --- etcd/cmd/microshift-etcd/run.go | 4 +- .../openshift/microshift/pkg/config/config.go | 40 +++++++++---------- pkg/cmd/init.go | 16 ++++---- pkg/cmd/run.go | 4 +- pkg/cmd/showConfig.go | 11 ++--- pkg/components/render.go | 4 +- pkg/config/config.go | 40 +++++++++---------- pkg/config/config_test.go | 32 ++++++++------- pkg/loadbalancerservice/controller.go | 2 +- pkg/mdns/controller.go | 4 +- pkg/node/kubelet.go | 4 +- pkg/sysconfwatch/sysconfwatch_linux.go | 2 +- 12 files changed, 78 insertions(+), 85 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index 0ea60ef6ec..4806b24323 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -92,8 +92,8 @@ func (s *EtcdService) configure(cfg *config.MicroshiftConfig) { s.etcdCfg.LCUrls = url2379 s.etcdCfg.ListenMetricsUrls = setURL([]string{"localhost"}, "2381") - s.etcdCfg.Name = cfg.NodeName - s.etcdCfg.InitialCluster = fmt.Sprintf("%s=https://%s:2380", cfg.NodeName, "localhost") + s.etcdCfg.Name = cfg.Node.HostnameOverride + s.etcdCfg.InitialCluster = fmt.Sprintf("%s=https://%s:2380", cfg.Node.HostnameOverride, "localhost") s.etcdCfg.CipherSuites = tlsCipherSuites s.etcdCfg.ClientTLSInfo.CertFile = cryptomaterial.PeerCertPath(etcdServingCertDir) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index e905540e03..4ce9a5acb4 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -76,8 +76,6 @@ type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` SubjectAltNames []string `json:"subjectAltNames"` - NodeName string `json:"nodeName"` - NodeIP string `json:"nodeIP"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into // the IP configured in the endpoint slice for kubernetes service. Must be @@ -92,7 +90,8 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` - DNS DNS `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` } // Top level config file @@ -248,8 +247,10 @@ func NewMicroshiftConfig() *MicroshiftConfig { return &MicroshiftConfig{ LogVLevel: 2, SubjectAltNames: subjectAltNames, - NodeName: strings.ToLower(nodeName), - NodeIP: nodeIP, + Node: Node{ + HostnameOverride: strings.ToLower(nodeName), + NodeIP: nodeIP, + }, DNS: DNS{ BaseDomain: "example.com", }, @@ -276,7 +277,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { if err != nil { klog.Fatalf("Failed to get hostname %v", err) } - return c.NodeName == strings.ToLower(hostname) + return c.Node.HostnameOverride == strings.ToLower(hostname) } // Read or set the NodeName that will be used for this MicroShift instance @@ -286,10 +287,10 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { if os.IsNotExist(err) { // ensure that dataDir exists os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.NodeName), 0444); err != nil { + if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) } - return c.NodeName, nil + return c.Node.HostnameOverride, nil } else if err != nil { return "", err } @@ -298,8 +299,8 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { // Validate the NodeName to be used for this MicroShift instances func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.NodeName); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.NodeName) + if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } establishedNodeName, err := c.establishNodeName() @@ -307,14 +308,14 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { return fmt.Errorf("failed to establish NodeName: %v", err) } - if establishedNodeName != c.NodeName { + if establishedNodeName != c.Node.HostnameOverride { if !isDefaultNodeName { return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.NodeName, establishedNodeName) + c.Node.HostnameOverride, establishedNodeName) } else { - c.NodeName = establishedNodeName + c.Node.HostnameOverride = establishedNodeName klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.NodeName) + "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) } } @@ -399,12 +400,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.LogVLevel = config.GetVerbosity() - if config.Node.HostnameOverride != "" { - c.NodeName = strings.ToLower(config.Node.HostnameOverride) - } - if config.Node.NodeIP != "" { - c.NodeIP = config.Node.NodeIP - } + c.Node = config.Node if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -495,10 +491,10 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.NodeIP) { + if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.NodeName { + if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 688dc8fcca..4e57e177de 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -71,7 +71,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } externalCertNames := []string{ - cfg.NodeName, + cfg.Node.HostnameOverride, "api." + cfg.DNS.BaseDomain, } externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) @@ -81,8 +81,8 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err // which certificate to serve which destination IP, internal pods start // getting the external certificate, which is signed by a different CA and // does not match the hostname. - if cfg.KASAdvertiseAddress != cfg.NodeIP { - externalCertNames = append(externalCertNames, cfg.NodeIP) + if cfg.KASAdvertiseAddress != cfg.Node.NodeIP { + externalCertNames = append(externalCertNames, cfg.Node.NodeIP) } certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) @@ -172,7 +172,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, // userinfo per https://kubernetes.io/docs/reference/access-authn-authz/node/#overview - UserInfo: &user.DefaultInfo{Name: "system:node:" + cfg.NodeName, Groups: []string{"system:nodes"}}, + UserInfo: &user.DefaultInfo{Name: "system:node:" + cfg.Node.HostnameOverride, Groups: []string{"system:nodes"}}, }, ).WithServingCertificates( &certchains.ServingCertificateSigningRequestInfo{ @@ -180,7 +180,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err Name: "kubelet-server", ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, - Hostnames: []string{cfg.NodeName, cfg.NodeIP}, + Hostnames: []string{cfg.Node.HostnameOverride, cfg.Node.NodeIP}, }, ), ), @@ -314,7 +314,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.LongLivedCertificateValidityDays, }, UserInfo: &user.DefaultInfo{Name: "system:etcd-peer:etcd-client", Groups: []string{"system:etcd-peers"}}, - Hostnames: []string{"localhost", cfg.NodeName}, + Hostnames: []string{"localhost", cfg.Node.HostnameOverride}, }, &certchains.PeerCertificateSigningRequestInfo{ CSRMeta: certchains.CSRMeta{ @@ -322,7 +322,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.LongLivedCertificateValidityDays, }, UserInfo: &user.DefaultInfo{Name: "system:etcd-server:etcd-client", Groups: []string{"system:etcd-servers"}}, - Hostnames: []string{"localhost", cfg.NodeName}, + Hostnames: []string{"localhost", cfg.Node.HostnameOverride}, }, ), ).WithCABundle( @@ -393,7 +393,7 @@ func initKubeconfigs( } // Generate one kubeconfigs per name - for _, name := range append(cfg.SubjectAltNames, cfg.NodeName) { + for _, name := range append(cfg.SubjectAltNames, cfg.Node.HostnameOverride) { u.Host = fmt.Sprintf("%s:%d", name, apiServerPort) if err := util.KubeConfigWithClientCerts( cfg.KubeConfigAdminPath(name), diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index ccc6c7fc94..6aeed47c95 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -56,8 +56,8 @@ func RunMicroshift(cfg *config.MicroshiftConfig) error { // see https://github.com/openshift/microshift/pull/471 if err := util.AddToNoProxyEnv( - cfg.NodeIP, - cfg.NodeName, + cfg.Node.NodeIP, + cfg.Node.HostnameOverride, cfg.Cluster.ClusterCIDR, cfg.Cluster.ServiceCIDR, ".svc", diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 1adfd5a7b0..dd0eb19bbb 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -31,8 +31,8 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command switch opts.Mode { case "default": - cfg.NodeIP = "" - cfg.NodeName = "" + cfg.Node.NodeIP = "" + cfg.Node.HostnameOverride = "" case "effective": // Load the current configuration if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { @@ -55,11 +55,8 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, }, - DNS: cfg.DNS, - Node: config.Node{ - HostnameOverride: cfg.NodeName, - NodeIP: cfg.NodeIP, - }, + DNS: cfg.DNS, + Node: cfg.Node, ApiServer: config.ApiServer{ SubjectAltNames: cfg.SubjectAltNames, }, diff --git a/pkg/components/render.go b/pkg/components/render.go index d826db28eb..3c63e62060 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -23,8 +23,8 @@ var templateFuncs = map[string]interface{}{ func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderParams) assets.RenderParams { params := map[string]interface{}{ "ReleaseImage": release.Image, - "NodeName": cfg.NodeName, - "NodeIP": cfg.NodeIP, + "NodeName": cfg.Node.HostnameOverride, + "NodeIP": cfg.Node.NodeIP, "ClusterCIDR": cfg.Cluster.ClusterCIDR, "ServiceCIDR": cfg.Cluster.ServiceCIDR, "ClusterDNS": cfg.Cluster.DNS, diff --git a/pkg/config/config.go b/pkg/config/config.go index e905540e03..4ce9a5acb4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -76,8 +76,6 @@ type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` SubjectAltNames []string `json:"subjectAltNames"` - NodeName string `json:"nodeName"` - NodeIP string `json:"nodeIP"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into // the IP configured in the endpoint slice for kubernetes service. Must be @@ -92,7 +90,8 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` - DNS DNS `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` } // Top level config file @@ -248,8 +247,10 @@ func NewMicroshiftConfig() *MicroshiftConfig { return &MicroshiftConfig{ LogVLevel: 2, SubjectAltNames: subjectAltNames, - NodeName: strings.ToLower(nodeName), - NodeIP: nodeIP, + Node: Node{ + HostnameOverride: strings.ToLower(nodeName), + NodeIP: nodeIP, + }, DNS: DNS{ BaseDomain: "example.com", }, @@ -276,7 +277,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { if err != nil { klog.Fatalf("Failed to get hostname %v", err) } - return c.NodeName == strings.ToLower(hostname) + return c.Node.HostnameOverride == strings.ToLower(hostname) } // Read or set the NodeName that will be used for this MicroShift instance @@ -286,10 +287,10 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { if os.IsNotExist(err) { // ensure that dataDir exists os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.NodeName), 0444); err != nil { + if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) } - return c.NodeName, nil + return c.Node.HostnameOverride, nil } else if err != nil { return "", err } @@ -298,8 +299,8 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { // Validate the NodeName to be used for this MicroShift instances func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.NodeName); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.NodeName) + if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } establishedNodeName, err := c.establishNodeName() @@ -307,14 +308,14 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { return fmt.Errorf("failed to establish NodeName: %v", err) } - if establishedNodeName != c.NodeName { + if establishedNodeName != c.Node.HostnameOverride { if !isDefaultNodeName { return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.NodeName, establishedNodeName) + c.Node.HostnameOverride, establishedNodeName) } else { - c.NodeName = establishedNodeName + c.Node.HostnameOverride = establishedNodeName klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.NodeName) + "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) } } @@ -399,12 +400,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.LogVLevel = config.GetVerbosity() - if config.Node.HostnameOverride != "" { - c.NodeName = strings.ToLower(config.Node.HostnameOverride) - } - if config.Node.NodeIP != "" { - c.NodeIP = config.Node.NodeIP - } + c.Node = config.Node if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -495,10 +491,10 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.NodeIP) { + if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.NodeName { + if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 78401276dd..797afb9d29 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -59,10 +59,12 @@ func TestConfigFile(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, - SubjectAltNames: []string{"node1", "node2"}, - NodeName: "node1", - NodeIP: "1.2.3.4", + LogVLevel: 4, + SubjectAltNames: []string{"node1", "node2"}, + Node: Node{ + HostnameOverride: "node1", + NodeIP: "1.2.3.4", + }, KASAdvertiseAddress: "6.7.8.9", DNS: DNS{ BaseDomain: "example.com", @@ -155,10 +157,12 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, - SubjectAltNames: []string{"node1", "node2"}, - NodeName: "node1", - NodeIP: "1.2.3.4", + LogVLevel: 4, + SubjectAltNames: []string{"node1", "node2"}, + Node: Node{ + HostnameOverride: "node1", + NodeIP: "1.2.3.4", + }, KASAdvertiseAddress: "6.7.8.9", SkipKASInterface: true, DNS: DNS{ @@ -239,7 +243,7 @@ func TestMicroshiftConfigIsDefaultNodeName(t *testing.T) { t.Errorf("expected default IsDefaultNodeName to be true") } - c.NodeName += "-suffix" + c.Node.HostnameOverride += "-suffix" if c.isDefaultNodeName() { t.Errorf("expected default IsDefaultNodeName to be false") } @@ -250,7 +254,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { defer cleanup() c := NewMicroshiftConfig() - c.NodeName = "node1" + c.Node.HostnameOverride = "node1" if err := c.validateNodeName(IS_NOT_DEFAULT_NODENAME); err != nil { t.Errorf("failed to validate node name on first call: %v", err) @@ -259,7 +263,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { nodeNameFile := filepath.Join(dataDir, ".nodename") if data, err := os.ReadFile(nodeNameFile); err != nil { t.Errorf("failed to read node name from file %q: %v", nodeNameFile, err) - } else if string(data) != c.NodeName { + } else if string(data) != c.Node.HostnameOverride { t.Errorf("node name file doesn't match the node name in the saved file: %v", err) } @@ -267,7 +271,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { t.Errorf("failed to validate node name on second call without changes: %v", err) } - c.NodeName = "node2" + c.Node.HostnameOverride = "node2" if err := c.validateNodeName(IS_NOT_DEFAULT_NODENAME); err == nil { t.Errorf("validation should have failed for nodename change: %v", err) } @@ -295,7 +299,7 @@ func TestMicroshiftConfigNodeNameValidationFromDefault(t *testing.T) { t.Errorf("failed to validate node name on second call without changes: %v", err) } - c.NodeName = "node2" + c.Node.HostnameOverride = "node2" if err := c.validateNodeName(IS_DEFAULT_NODENAME); err != nil { t.Errorf("validation should have failed in this case, it must be a warning in logs: %v", err) } @@ -306,7 +310,7 @@ func TestMicroshiftConfigNodeNameValidationBadName(t *testing.T) { defer cleanup() c := NewMicroshiftConfig() - c.NodeName = "1.2.3.4" + c.Node.HostnameOverride = "1.2.3.4" if err := c.validateNodeName(IS_DEFAULT_NODENAME); err == nil { t.Errorf("failed to validate node name.") diff --git a/pkg/loadbalancerservice/controller.go b/pkg/loadbalancerservice/controller.go index 56fd397051..2a5d9a0ad3 100644 --- a/pkg/loadbalancerservice/controller.go +++ b/pkg/loadbalancerservice/controller.go @@ -36,7 +36,7 @@ var _ servicemanager.Service = &LoadbalancerServiceController{} func NewLoadbalancerServiceController(cfg *config.MicroshiftConfig) *LoadbalancerServiceController { return &LoadbalancerServiceController{ - NodeIP: cfg.NodeIP, + NodeIP: cfg.Node.NodeIP, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), } } diff --git a/pkg/mdns/controller.go b/pkg/mdns/controller.go index 7ff7e62b9b..3daee405c2 100644 --- a/pkg/mdns/controller.go +++ b/pkg/mdns/controller.go @@ -25,8 +25,8 @@ type MicroShiftmDNSController struct { func NewMicroShiftmDNSController(cfg *config.MicroshiftConfig) *MicroShiftmDNSController { return &MicroShiftmDNSController{ - NodeIP: cfg.NodeIP, - NodeName: cfg.NodeName, + NodeIP: cfg.Node.NodeIP, + NodeName: cfg.Node.HostnameOverride, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), hostCount: make(map[string]int), } diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index 562c49dad8..d9667e738b 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -74,8 +74,8 @@ func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { kubeletFlags.BootstrapKubeconfig = cfg.KubeConfigPath(config.Kubelet) kubeletFlags.KubeConfig = cfg.KubeConfigPath(config.Kubelet) kubeletFlags.RuntimeCgroups = "/system.slice/crio.service" - kubeletFlags.HostnameOverride = cfg.NodeName - kubeletFlags.NodeIP = cfg.NodeIP + kubeletFlags.HostnameOverride = cfg.Node.HostnameOverride + kubeletFlags.NodeIP = cfg.Node.NodeIP kubeletFlags.ContainerRuntime = "remote" kubeletFlags.RemoteRuntimeEndpoint = "unix:///var/run/crio/crio.sock" kubeletFlags.NodeLabels["node-role.kubernetes.io/control-plane"] = "" diff --git a/pkg/sysconfwatch/sysconfwatch_linux.go b/pkg/sysconfwatch/sysconfwatch_linux.go index b68d159bd6..0c18891ff7 100644 --- a/pkg/sysconfwatch/sysconfwatch_linux.go +++ b/pkg/sysconfwatch/sysconfwatch_linux.go @@ -55,7 +55,7 @@ func NewSysConfWatchController(cfg *config.MicroshiftConfig) *SysConfWatchContro } return &SysConfWatchController{ - NodeIP: cfg.NodeIP, + NodeIP: cfg.Node.NodeIP, timerFd: fd, } } From e6cce6eb47ffa82b4ea96fd6e53567c7efbbe855 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 16:56:12 -0500 Subject: [PATCH 03/17] USHIFT-936: align the Debugging portion of the config data structures --- .../openshift/microshift/pkg/config/config.go | 15 ++++++++------- pkg/cmd/showConfig.go | 9 +-------- pkg/config/config.go | 15 ++++++++------- pkg/config/config_test.go | 8 ++++++-- pkg/controllers/kube-apiserver.go | 2 +- pkg/controllers/kube-controller-manager.go | 2 +- 6 files changed, 25 insertions(+), 26 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 4ce9a5acb4..9e539aab1b 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -73,8 +73,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - LogVLevel int `json:"logVLevel"` - SubjectAltNames []string `json:"subjectAltNames"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into @@ -90,8 +88,9 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` - DNS DNS `json:"-"` - Node Node `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` + Debugging Debugging `json:"debugging"` } // Top level config file @@ -245,7 +244,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { } return &MicroshiftConfig{ - LogVLevel: 2, + Debugging: Debugging{ + LogLevel: "Normal", + }, SubjectAltNames: subjectAltNames, Node: Node{ HostnameOverride: strings.ToLower(nodeName), @@ -399,8 +400,8 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { } // Wire new Config type to existing MicroshiftConfig - c.LogVLevel = config.GetVerbosity() c.Node = config.Node + c.Debugging = config.Debugging if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -550,7 +551,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { +func (c *MicroshiftConfig) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index dd0eb19bbb..ed0bdc32c0 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -5,7 +5,6 @@ import ( "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/yaml" @@ -43,10 +42,6 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command } // map back from internal representation to user config - logLevels := []string{"", "", "Normal", "", "Debug", "", "Trace", "", "TraceAll"} - if cfg.LogVLevel < 0 || cfg.LogVLevel >= len(logLevels) { - klog.Fatal("logVLevel out of range [0..%d] %d", len(logLevels)-1, cfg.LogVLevel) - } userCfg := config.Config{ Network: config.Network{ ClusterNetwork: []config.ClusterNetworkEntry{ @@ -60,9 +55,7 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ApiServer: config.ApiServer{ SubjectAltNames: cfg.SubjectAltNames, }, - Debugging: config.Debugging{ - LogLevel: logLevels[cfg.LogVLevel], - }, + Debugging: cfg.Debugging, } marshalled, err := yaml.Marshal(userCfg) cmdutil.CheckErr(err) diff --git a/pkg/config/config.go b/pkg/config/config.go index 4ce9a5acb4..9e539aab1b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -73,8 +73,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - LogVLevel int `json:"logVLevel"` - SubjectAltNames []string `json:"subjectAltNames"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into @@ -90,8 +88,9 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` - DNS DNS `json:"-"` - Node Node `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` + Debugging Debugging `json:"debugging"` } // Top level config file @@ -245,7 +244,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { } return &MicroshiftConfig{ - LogVLevel: 2, + Debugging: Debugging{ + LogLevel: "Normal", + }, SubjectAltNames: subjectAltNames, Node: Node{ HostnameOverride: strings.ToLower(nodeName), @@ -399,8 +400,8 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { } // Wire new Config type to existing MicroshiftConfig - c.LogVLevel = config.GetVerbosity() c.Node = config.Node + c.Debugging = config.Debugging if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -550,7 +551,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { +func (c *MicroshiftConfig) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 797afb9d29..186cb16c55 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -59,7 +59,9 @@ func TestConfigFile(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, + Debugging: Debugging{ + LogLevel: "Debug", + }, SubjectAltNames: []string{"node1", "node2"}, Node: Node{ HostnameOverride: "node1", @@ -157,7 +159,9 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, + Debugging: Debugging{ + LogLevel: "Debug", + }, SubjectAltNames: []string{"node1", "node2"}, Node: Node{ HostnameOverride: "node1", diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index a1801332b5..cb73cd8d98 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -92,7 +92,7 @@ func (s *KubeAPIServer) Name() string { return "kube-apiserver" } func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd", "network-configuration"} } func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { - s.verbosity = cfg.LogVLevel + s.verbosity = cfg.GetVerbosity() certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) kubeCSRSignerDir := cryptomaterial.CSRSignerCertDir(certsDir) diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index a9e6409e04..232e346377 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -93,7 +93,7 @@ func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() erro "use-service-account-credentials": {"true"}, "cluster-signing-cert-file": {clusterSigningCert}, "cluster-signing-key-file": {clusterSigningKey}, - "v": {strconv.Itoa(cfg.LogVLevel)}, + "v": {strconv.Itoa(cfg.GetVerbosity())}, }, } From 23523cf98dbc1bd098b4e7584938fe2972553cf6 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 17:11:24 -0500 Subject: [PATCH 04/17] USHIFT-936: align the Etcd portion of the config data structures --- .../openshift/microshift/pkg/config/config.go | 15 +++++++++------ pkg/cmd/showConfig.go | 3 +++ pkg/config/config.go | 15 +++++++++------ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 9e539aab1b..3e83391f9c 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -59,17 +59,20 @@ type IngressConfig struct { type EtcdConfig struct { // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimit uint64 + MemoryLimit uint64 `json:"memoryLimit"` + // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendBytes int64 + QuotaBackendBytes int64 `json:"-"` + // If the backend is fragmented more than `maxFragmentedPercentage` // and the database size is greater than `minDefragBytes`, do a defrag. - MinDefragBytes int64 - MaxFragmentedPercentage float64 + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"-"` + // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq time.Duration + DefragCheckFreq time.Duration `json:"-"` // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool + DoStartupDefrag bool `json:"-"` } type MicroshiftConfig struct { diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index ed0bdc32c0..51e3e46f56 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -56,6 +56,9 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command SubjectAltNames: cfg.SubjectAltNames, }, Debugging: cfg.Debugging, + Etcd: config.Etcd{ + MemoryLimitMB: cfg.Etcd.MemoryLimit, + }, } marshalled, err := yaml.Marshal(userCfg) cmdutil.CheckErr(err) diff --git a/pkg/config/config.go b/pkg/config/config.go index 9e539aab1b..3e83391f9c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -59,17 +59,20 @@ type IngressConfig struct { type EtcdConfig struct { // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimit uint64 + MemoryLimit uint64 `json:"memoryLimit"` + // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendBytes int64 + QuotaBackendBytes int64 `json:"-"` + // If the backend is fragmented more than `maxFragmentedPercentage` // and the database size is greater than `minDefragBytes`, do a defrag. - MinDefragBytes int64 - MaxFragmentedPercentage float64 + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"-"` + // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq time.Duration + DefragCheckFreq time.Duration `json:"-"` // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool + DoStartupDefrag bool `json:"-"` } type MicroshiftConfig struct { From ce65978380887be422d34e5cca72de80d576aaef Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 17:24:05 -0500 Subject: [PATCH 05/17] USHIFT-936: align the ApiServer portion of the config data structures --- .../openshift/microshift/pkg/config/config.go | 54 +++++++++---------- pkg/cmd/init.go | 6 +-- pkg/cmd/showConfig.go | 8 ++- pkg/config/config.go | 54 +++++++++---------- pkg/config/config_test.go | 14 +++-- pkg/controllers/kube-apiserver.go | 2 +- pkg/node/netconfig.go | 4 +- 7 files changed, 66 insertions(+), 76 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 3e83391f9c..3033758c5a 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -76,17 +76,7 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - SubjectAltNames []string `json:"subjectAltNames"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - KASAdvertiseAddress string `json:"kasAdvertiseAddress"` - // Determines if kube-apiserver controller should configure the - // KASAdvertiseAddress in the loopback interface. Automatically computed. - SkipKASInterface bool `json:"-"` - Cluster ClusterConfig `json:"cluster"` + Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -94,6 +84,7 @@ type MicroshiftConfig struct { DNS DNS `json:"-"` Node Node `json:"-"` Debugging Debugging `json:"debugging"` + ApiServer ApiServer `json:"-"` } // Top level config file @@ -157,9 +148,15 @@ type DNS struct { type ApiServer struct { // SubjectAltNames added to API server certs SubjectAltNames []string `json:"subjectAltNames"` - // AdvertiseAddress for endpoint slices in kubernetes service. Developer - // only parameter, wont show in show-config commands or docs. + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` } type Node struct { @@ -250,7 +247,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { Debugging: Debugging{ LogLevel: "Normal", }, - SubjectAltNames: subjectAltNames, + ApiServer: ApiServer{ + SubjectAltNames: subjectAltNames, + }, Node: Node{ HostnameOverride: strings.ToLower(nodeName), NodeIP: nodeIP, @@ -415,12 +414,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } c.DNS = config.DNS - if len(config.ApiServer.SubjectAltNames) > 0 { - c.SubjectAltNames = config.ApiServer.SubjectAltNames - } - if len(config.ApiServer.AdvertiseAddress) > 0 { - c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress - } + c.ApiServer = config.ApiServer if config.Etcd.MemoryLimitMB > 0 { // If the memory limit is than the minimum, set it to the minimum and continue. @@ -452,7 +446,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // If KAS advertise address is not configured then compute it from the service // CIDR automatically. - if len(c.KASAdvertiseAddress) == 0 { + if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) // Since the KAS advertise address was not provided we will default to the @@ -467,13 +461,13 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { } // First and last are the same because of the /32 netmask. firstValidIP, _ := cidr.AddressRange(nextSubnet) - c.KASAdvertiseAddress = firstValidIP.String() - c.SkipKASInterface = false + c.ApiServer.AdvertiseAddress = firstValidIP.String() + c.ApiServer.SkipInterface = false } else { - c.SkipKASInterface = true + c.ApiServer.SkipInterface = true } - if len(c.SubjectAltNames) > 0 { + if len(c.ApiServer.SubjectAltNames) > 0 { // Any entry in SubjectAltNames will be included in the external access certificates. // Any of the hostnames and IPs (except the node IP) listed below conflicts with // other certificates, such as the service network and localhost access. @@ -491,20 +485,20 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("failed to parse cluster URL: %v", err) } if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.SubjectAltNames, "localhost", "127.0.0.1") { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } if stringSliceContains( - c.SubjectAltNames, + c.ApiServer.SubjectAltNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc", @@ -513,7 +507,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - c.KASAdvertiseAddress, + c.ApiServer.AdvertiseAddress, ) { return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 4e57e177de..8473d833ca 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -74,14 +74,14 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err cfg.Node.HostnameOverride, "api." + cfg.DNS.BaseDomain, } - externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) + externalCertNames = append(externalCertNames, cfg.ApiServer.SubjectAltNames...) // When Kube apiserver advertise address matches the node IP we can not add // it to the certificates or else the internal pod access to apiserver is // broken. Because of client-go not using SNI and the way apiserver handles // which certificate to serve which destination IP, internal pods start // getting the external certificate, which is signed by a different CA and // does not match the hostname. - if cfg.KASAdvertiseAddress != cfg.Node.NodeIP { + if cfg.ApiServer.AdvertiseAddress != cfg.Node.NodeIP { externalCertNames = append(externalCertNames, cfg.Node.NodeIP) } @@ -393,7 +393,7 @@ func initKubeconfigs( } // Generate one kubeconfigs per name - for _, name := range append(cfg.SubjectAltNames, cfg.Node.HostnameOverride) { + for _, name := range append(cfg.ApiServer.SubjectAltNames, cfg.Node.HostnameOverride) { u.Host = fmt.Sprintf("%s:%d", name, apiServerPort) if err := util.KubeConfigWithClientCerts( cfg.KubeConfigAdminPath(name), diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 51e3e46f56..f500917f5e 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -50,11 +50,9 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, }, - DNS: cfg.DNS, - Node: cfg.Node, - ApiServer: config.ApiServer{ - SubjectAltNames: cfg.SubjectAltNames, - }, + DNS: cfg.DNS, + Node: cfg.Node, + ApiServer: cfg.ApiServer, Debugging: cfg.Debugging, Etcd: config.Etcd{ MemoryLimitMB: cfg.Etcd.MemoryLimit, diff --git a/pkg/config/config.go b/pkg/config/config.go index 3e83391f9c..3033758c5a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -76,17 +76,7 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - SubjectAltNames []string `json:"subjectAltNames"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - KASAdvertiseAddress string `json:"kasAdvertiseAddress"` - // Determines if kube-apiserver controller should configure the - // KASAdvertiseAddress in the loopback interface. Automatically computed. - SkipKASInterface bool `json:"-"` - Cluster ClusterConfig `json:"cluster"` + Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -94,6 +84,7 @@ type MicroshiftConfig struct { DNS DNS `json:"-"` Node Node `json:"-"` Debugging Debugging `json:"debugging"` + ApiServer ApiServer `json:"-"` } // Top level config file @@ -157,9 +148,15 @@ type DNS struct { type ApiServer struct { // SubjectAltNames added to API server certs SubjectAltNames []string `json:"subjectAltNames"` - // AdvertiseAddress for endpoint slices in kubernetes service. Developer - // only parameter, wont show in show-config commands or docs. + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` } type Node struct { @@ -250,7 +247,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { Debugging: Debugging{ LogLevel: "Normal", }, - SubjectAltNames: subjectAltNames, + ApiServer: ApiServer{ + SubjectAltNames: subjectAltNames, + }, Node: Node{ HostnameOverride: strings.ToLower(nodeName), NodeIP: nodeIP, @@ -415,12 +414,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } c.DNS = config.DNS - if len(config.ApiServer.SubjectAltNames) > 0 { - c.SubjectAltNames = config.ApiServer.SubjectAltNames - } - if len(config.ApiServer.AdvertiseAddress) > 0 { - c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress - } + c.ApiServer = config.ApiServer if config.Etcd.MemoryLimitMB > 0 { // If the memory limit is than the minimum, set it to the minimum and continue. @@ -452,7 +446,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // If KAS advertise address is not configured then compute it from the service // CIDR automatically. - if len(c.KASAdvertiseAddress) == 0 { + if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) // Since the KAS advertise address was not provided we will default to the @@ -467,13 +461,13 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { } // First and last are the same because of the /32 netmask. firstValidIP, _ := cidr.AddressRange(nextSubnet) - c.KASAdvertiseAddress = firstValidIP.String() - c.SkipKASInterface = false + c.ApiServer.AdvertiseAddress = firstValidIP.String() + c.ApiServer.SkipInterface = false } else { - c.SkipKASInterface = true + c.ApiServer.SkipInterface = true } - if len(c.SubjectAltNames) > 0 { + if len(c.ApiServer.SubjectAltNames) > 0 { // Any entry in SubjectAltNames will be included in the external access certificates. // Any of the hostnames and IPs (except the node IP) listed below conflicts with // other certificates, such as the service network and localhost access. @@ -491,20 +485,20 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("failed to parse cluster URL: %v", err) } if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.SubjectAltNames, "localhost", "127.0.0.1") { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } if stringSliceContains( - c.SubjectAltNames, + c.ApiServer.SubjectAltNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc", @@ -513,7 +507,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - c.KASAdvertiseAddress, + c.ApiServer.AdvertiseAddress, ) { return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 186cb16c55..c4f6794e92 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -62,12 +62,14 @@ func TestConfigFile(t *testing.T) { Debugging: Debugging{ LogLevel: "Debug", }, - SubjectAltNames: []string{"node1", "node2"}, + ApiServer: ApiServer{ + SubjectAltNames: []string{"node1", "node2"}, + AdvertiseAddress: "6.7.8.9", + }, Node: Node{ HostnameOverride: "node1", NodeIP: "1.2.3.4", }, - KASAdvertiseAddress: "6.7.8.9", DNS: DNS{ BaseDomain: "example.com", }, @@ -162,13 +164,15 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { Debugging: Debugging{ LogLevel: "Debug", }, - SubjectAltNames: []string{"node1", "node2"}, + ApiServer: ApiServer{ + SubjectAltNames: []string{"node1", "node2"}, + AdvertiseAddress: "6.7.8.9", + SkipInterface: true, + }, Node: Node{ HostnameOverride: "node1", NodeIP: "1.2.3.4", }, - KASAdvertiseAddress: "6.7.8.9", - SkipKASInterface: true, DNS: DNS{ BaseDomain: "example.com", }, diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index cb73cd8d98..7044af138c 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -117,7 +117,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { s.masterURL = cfg.Cluster.URL s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir) - s.advertiseAddress = cfg.KASAdvertiseAddress + s.advertiseAddress = cfg.ApiServer.AdvertiseAddress overrides := &kubecontrolplanev1.KubeAPIServerConfig{ APIServerArguments: map[string]kubecontrolplanev1.Arguments{ diff --git a/pkg/node/netconfig.go b/pkg/node/netconfig.go index e43813d7f8..e92f9c81d3 100644 --- a/pkg/node/netconfig.go +++ b/pkg/node/netconfig.go @@ -48,8 +48,8 @@ func (n *NetworkConfiguration) Name() string { return componentNetwork func (n *NetworkConfiguration) Dependencies() []string { return []string{} } func (n *NetworkConfiguration) configure(cfg *config.MicroshiftConfig) { - n.kasAdvertiseAddress = cfg.KASAdvertiseAddress - n.skipInterfaceConfiguration = cfg.SkipKASInterface + n.kasAdvertiseAddress = cfg.ApiServer.AdvertiseAddress + n.skipInterfaceConfiguration = cfg.ApiServer.SkipInterface } func (n *NetworkConfiguration) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error { From 296ffed9c4907a94241d48687d59e6cfdaad51e8 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 18:23:53 -0500 Subject: [PATCH 06/17] USHIFT-936: align the Cluster/Network portions of the config data structures Also introduces assert package in the test code for easier diagnosis of unit test failures. --- .../openshift/microshift/pkg/config/config.go | 73 +++++++++++-------- pkg/cmd/init.go | 18 ++--- pkg/cmd/run.go | 4 +- pkg/cmd/showConfig.go | 6 +- pkg/components/controllers.go | 2 +- pkg/components/render.go | 6 +- pkg/config/config.go | 73 +++++++++++-------- pkg/config/config_test.go | 36 +++++---- pkg/controllers/kube-apiserver.go | 10 +-- pkg/controllers/kube-controller-manager.go | 2 +- pkg/node/kubelet.go | 2 +- 11 files changed, 132 insertions(+), 100 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 3033758c5a..3fe7145e12 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -44,14 +44,6 @@ var ( manifestsDir = findManifestsDir() ) -type ClusterConfig struct { - URL string `json:"-"` - ClusterCIDR string `json:"clusterCIDR"` - ServiceCIDR string `json:"serviceCIDR"` - ServiceNodePortRange string `json:"serviceNodePortRange"` - DNS string `json:"-"` -} - type IngressConfig struct { ServingCertificate []byte ServingKey []byte @@ -76,8 +68,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -85,6 +75,7 @@ type MicroshiftConfig struct { Node Node `json:"-"` Debugging Debugging `json:"debugging"` ApiServer ApiServer `json:"-"` + Network Network `json:"-"` } // Top level config file @@ -125,6 +116,9 @@ type Network struct { // installed. // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` + + // The DNS server to use + DNS string `json:"-"` } type ClusterNetworkEntry struct { @@ -157,6 +151,9 @@ type ApiServer struct { // Determines if kube-apiserver controller should configure the // AdvertiseAddress in the loopback interface. Automatically computed. SkipInterface bool `json:"-"` + + // The URL of the API server + URL string `json:"-"` } type Node struct { @@ -249,6 +246,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { }, ApiServer: ApiServer{ SubjectAltNames: subjectAltNames, + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: strings.ToLower(nodeName), @@ -257,11 +255,17 @@ func NewMicroshiftConfig() *MicroshiftConfig { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.42.0.0/16", - ServiceCIDR: "10.43.0.0/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + }, + ServiceNetwork: []string{ + "10.43.0.0/16", + }, ServiceNodePortRange: "30000-32767", + DNS: "10.43.0.10", }, Etcd: EtcdConfig{ MemoryLimit: 0, // No limit @@ -326,10 +330,10 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *ClusterConfig) ApiServerPort() (int, error) { +func (c *MicroshiftConfig) ApiServerPort() (int, error) { var port string - parsed, err := url.Parse(c.URL) + parsed, err := url.Parse(c.ApiServer.URL) if err != nil { return 0, err } @@ -404,17 +408,14 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.Node = config.Node c.Debugging = config.Debugging - if len(config.Network.ClusterNetwork) != 0 { - c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR - } - if len(config.Network.ServiceNetwork) != 0 { - c.Cluster.ServiceCIDR = config.Network.ServiceNetwork[0] - } - if config.Network.ServiceNodePortRange != "" { - c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange + c.Network = config.Network + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } + c.DNS = config.DNS c.ApiServer = config.ApiServer + c.ApiServer.URL = "https://localhost:6443" if config.Etcd.MemoryLimitMB > 0 { // If the memory limit is than the minimum, set it to the minimum and continue. @@ -428,6 +429,19 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return nil } +func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { + if len(c.Network.ServiceNetwork) == 0 { + return fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to get DNS IP: %v", err) + } + c.Network.DNS = clusterDNS + return nil +} + // Note: add a configFile parameter here because of unit test requiring custom // local directory func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { @@ -437,18 +451,15 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { } } - // validate serviceCIDR - clusterDNS, err := getClusterDNS(c.Cluster.ServiceCIDR) - if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } - c.Cluster.DNS = clusterDNS // If KAS advertise address is not configured then compute it from the service // CIDR automatically. if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) // Since the KAS advertise address was not provided we will default to the // next immediate subnet after the service CIDR. This is due to the fact // that using the actual apiserver service IP as an endpoint slice breaks @@ -480,7 +491,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // the node IP it returns that certificate, which is the external access one. This // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate // is invalid. - u, err := url.Parse(c.Cluster.URL) + u, err := url.Parse(c.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 8473d833ca..3023036d8b 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -60,7 +60,7 @@ func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { - _, svcNet, err := net.ParseCIDR(cfg.Cluster.ServiceCIDR) + _, svcNet, err := net.ParseCIDR(cfg.Network.ServiceNetwork[0]) if err != nil { return nil, err } @@ -383,11 +383,11 @@ func initKubeconfigs( return err } - u, err := url.Parse(cfg.Cluster.URL) + u, err := url.Parse(cfg.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } - apiServerPort, err := cfg.Cluster.ApiServerPort() + apiServerPort, err := cfg.ApiServerPort() if err != nil { return fmt.Errorf("failed to get apiserver port: %v", err) } @@ -408,7 +408,7 @@ func initKubeconfigs( if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeAdmin), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, adminKubeconfigCertPEM, adminKubeconfigKeyPEM, @@ -422,7 +422,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeControllerManager), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, kcmCertPEM, kcmKeyPEM, @@ -436,7 +436,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeScheduler), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, schedulerCertPEM, schedulerKeyPEM, ); err != nil { @@ -449,7 +449,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.Kubelet), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, kubeletCertPEM, kubeletKeyPEM, ); err != nil { @@ -461,7 +461,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.ClusterPolicyController), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, clusterPolicyControllerCertPEM, clusterPolicyControllerKeyPEM, ); err != nil { @@ -474,7 +474,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.RouteControllerManager), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, routeControllerManagerCertPEM, routeControllerManagerKeyPEM, ); err != nil { diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 6aeed47c95..5ae200ba0d 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -58,8 +58,8 @@ func RunMicroshift(cfg *config.MicroshiftConfig) error { if err := util.AddToNoProxyEnv( cfg.Node.NodeIP, cfg.Node.HostnameOverride, - cfg.Cluster.ClusterCIDR, - cfg.Cluster.ServiceCIDR, + cfg.Network.ClusterNetwork[0].CIDR, + cfg.Network.ServiceNetwork[0], ".svc", ".cluster.local", "."+cfg.DNS.BaseDomain); err != nil { diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index f500917f5e..8ea011ec65 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -45,10 +45,10 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command userCfg := config.Config{ Network: config.Network{ ClusterNetwork: []config.ClusterNetworkEntry{ - {CIDR: cfg.Cluster.ClusterCIDR}, + {CIDR: cfg.Network.ClusterNetwork[0].CIDR}, }, - ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, - ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, + ServiceNetwork: []string{cfg.Network.ServiceNetwork[0]}, + ServiceNodePortRange: cfg.Network.ServiceNodePortRange, }, DNS: cfg.DNS, Node: cfg.Node, diff --git a/pkg/components/controllers.go b/pkg/components/controllers.go index 10ae6593c1..b69f965cf7 100644 --- a/pkg/components/controllers.go +++ b/pkg/components/controllers.go @@ -210,7 +210,7 @@ func startDNSController(cfg *config.MicroshiftConfig, kubeconfigPath string) err } extraParams := assets.RenderParams{ - "ClusterIP": cfg.Cluster.DNS, + "ClusterIP": cfg.Network.DNS, } if err := assets.ApplyServices(svc, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply service %v %v", svc, err) diff --git a/pkg/components/render.go b/pkg/components/render.go index 3c63e62060..b2f5b0dbce 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -25,9 +25,9 @@ func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderPar "ReleaseImage": release.Image, "NodeName": cfg.Node.HostnameOverride, "NodeIP": cfg.Node.NodeIP, - "ClusterCIDR": cfg.Cluster.ClusterCIDR, - "ServiceCIDR": cfg.Cluster.ServiceCIDR, - "ClusterDNS": cfg.Cluster.DNS, + "ClusterCIDR": cfg.Network.ClusterNetwork[0].CIDR, + "ServiceCIDR": cfg.Network.ServiceNetwork[0], + "ClusterDNS": cfg.Network.DNS, "BaseDomain": cfg.DNS.BaseDomain, } for k, v := range extra { diff --git a/pkg/config/config.go b/pkg/config/config.go index 3033758c5a..3fe7145e12 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -44,14 +44,6 @@ var ( manifestsDir = findManifestsDir() ) -type ClusterConfig struct { - URL string `json:"-"` - ClusterCIDR string `json:"clusterCIDR"` - ServiceCIDR string `json:"serviceCIDR"` - ServiceNodePortRange string `json:"serviceNodePortRange"` - DNS string `json:"-"` -} - type IngressConfig struct { ServingCertificate []byte ServingKey []byte @@ -76,8 +68,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -85,6 +75,7 @@ type MicroshiftConfig struct { Node Node `json:"-"` Debugging Debugging `json:"debugging"` ApiServer ApiServer `json:"-"` + Network Network `json:"-"` } // Top level config file @@ -125,6 +116,9 @@ type Network struct { // installed. // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` + + // The DNS server to use + DNS string `json:"-"` } type ClusterNetworkEntry struct { @@ -157,6 +151,9 @@ type ApiServer struct { // Determines if kube-apiserver controller should configure the // AdvertiseAddress in the loopback interface. Automatically computed. SkipInterface bool `json:"-"` + + // The URL of the API server + URL string `json:"-"` } type Node struct { @@ -249,6 +246,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { }, ApiServer: ApiServer{ SubjectAltNames: subjectAltNames, + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: strings.ToLower(nodeName), @@ -257,11 +255,17 @@ func NewMicroshiftConfig() *MicroshiftConfig { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.42.0.0/16", - ServiceCIDR: "10.43.0.0/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + }, + ServiceNetwork: []string{ + "10.43.0.0/16", + }, ServiceNodePortRange: "30000-32767", + DNS: "10.43.0.10", }, Etcd: EtcdConfig{ MemoryLimit: 0, // No limit @@ -326,10 +330,10 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *ClusterConfig) ApiServerPort() (int, error) { +func (c *MicroshiftConfig) ApiServerPort() (int, error) { var port string - parsed, err := url.Parse(c.URL) + parsed, err := url.Parse(c.ApiServer.URL) if err != nil { return 0, err } @@ -404,17 +408,14 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.Node = config.Node c.Debugging = config.Debugging - if len(config.Network.ClusterNetwork) != 0 { - c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR - } - if len(config.Network.ServiceNetwork) != 0 { - c.Cluster.ServiceCIDR = config.Network.ServiceNetwork[0] - } - if config.Network.ServiceNodePortRange != "" { - c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange + c.Network = config.Network + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } + c.DNS = config.DNS c.ApiServer = config.ApiServer + c.ApiServer.URL = "https://localhost:6443" if config.Etcd.MemoryLimitMB > 0 { // If the memory limit is than the minimum, set it to the minimum and continue. @@ -428,6 +429,19 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return nil } +func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { + if len(c.Network.ServiceNetwork) == 0 { + return fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to get DNS IP: %v", err) + } + c.Network.DNS = clusterDNS + return nil +} + // Note: add a configFile parameter here because of unit test requiring custom // local directory func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { @@ -437,18 +451,15 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { } } - // validate serviceCIDR - clusterDNS, err := getClusterDNS(c.Cluster.ServiceCIDR) - if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } - c.Cluster.DNS = clusterDNS // If KAS advertise address is not configured then compute it from the service // CIDR automatically. if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) // Since the KAS advertise address was not provided we will default to the // next immediate subnet after the service CIDR. This is due to the fact // that using the actual apiserver service IP as an endpoint slice breaks @@ -480,7 +491,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // the node IP it returns that certificate, which is the external access one. This // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate // is invalid. - u, err := url.Parse(c.Cluster.URL) + u, err := url.Parse(c.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index c4f6794e92..b91f9afdad 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -3,11 +3,12 @@ package config import ( "os" "path/filepath" - "reflect" "testing" "time" "sigs.k8s.io/yaml" + + "github.com/stretchr/testify/assert" ) const ( @@ -65,6 +66,7 @@ func TestConfigFile(t *testing.T) { ApiServer: ApiServer{ SubjectAltNames: []string{"node1", "node2"}, AdvertiseAddress: "6.7.8.9", + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: "node1", @@ -73,11 +75,15 @@ func TestConfigFile(t *testing.T) { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.20.30.40/16", - ServiceCIDR: "40.30.20.10/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.20.30.40/16", + }, + }, + ServiceNetwork: []string{"40.30.20.10/16"}, ServiceNodePortRange: "1024-32767", + DNS: "40.30.0.10", }, Etcd: EtcdConfig{ MemoryLimit: 0, @@ -114,8 +120,8 @@ func TestConfigFile(t *testing.T) { if !tt.expectErr && err != nil { t.Fatalf("Not expecting error and received: %v", err) } - if !tt.expectErr && !reflect.DeepEqual(*config, tt.expected) { - t.Errorf("ReadFromConfigFile() mismatch. got=%v, want=%v", *config, tt.expected) + if !tt.expectErr { + assert.Equal(t, tt.expected, *config) } }) } @@ -168,6 +174,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { SubjectAltNames: []string{"node1", "node2"}, AdvertiseAddress: "6.7.8.9", SkipInterface: true, + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: "node1", @@ -176,10 +183,13 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.20.30.40/16", - ServiceCIDR: "40.30.20.10/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.20.30.40/16", + }, + }, + ServiceNetwork: []string{"40.30.20.10/16"}, ServiceNodePortRange: "1024-32767", DNS: "40.30.0.10", }, @@ -238,8 +248,8 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { if !tt.expectErr && err != nil { t.Fatalf("Not expecting error and received: %v", err) } - if !tt.expectErr && !reflect.DeepEqual(*config, tt.expected) { - t.Errorf("ReadAndValidate() mismatch. got=%v, want=%v", *config, tt.expected) + if !tt.expectErr { + assert.Equal(t, tt.expected, *config) } }) } diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index 7044af138c..88ec80b3b3 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -110,12 +110,12 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { } // Get the apiserver port so we can set it as an argument - apiServerPort, err := cfg.Cluster.ApiServerPort() + apiServerPort, err := cfg.ApiServerPort() if err != nil { return err } - s.masterURL = cfg.Cluster.URL + s.masterURL = cfg.ApiServer.URL s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir) s.advertiseAddress = cfg.ApiServer.AdvertiseAddress @@ -139,7 +139,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { "proxy-client-key-file": {cryptomaterial.ClientKeyPath(aggregatorClientCertDir)}, "requestheader-client-ca-file": {aggregatorCAPath}, "service-account-signing-key-file": {microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.key"}, - "service-node-port-range": {cfg.Cluster.ServiceNodePortRange}, + "service-node-port-range": {cfg.Network.ServiceNodePortRange}, "tls-cert-file": {servingCert}, "tls-private-key-file": {servingKey}, "disable-admission-plugins": { @@ -214,8 +214,8 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { ServiceAccountPublicKeyFiles: []string{ microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.pub", }, - ServicesSubnet: cfg.Cluster.ServiceCIDR, - ServicesNodePortRange: cfg.Cluster.ServiceNodePortRange, + ServicesSubnet: cfg.Network.ServiceNetwork[0], + ServicesNodePortRange: cfg.Network.ServiceNodePortRange, } overridesBytes, err := json.Marshal(overrides) diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 232e346377..03e9ff30d6 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -85,7 +85,7 @@ func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() erro "authorization-kubeconfig": {kubeConfig}, "service-account-private-key-file": {kcmServiceAccountPrivateKeyFile()}, "allocate-node-cidrs": {"true"}, - "cluster-cidr": {cfg.Cluster.ClusterCIDR}, + "cluster-cidr": {cfg.Network.ClusterNetwork[0].CIDR}, "root-ca-file": {kcmRootCAFile()}, "bind-address": {"127.0.0.1"}, "secure-port": {"10257"}, diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index d9667e738b..1192c1f784 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -111,7 +111,7 @@ cgroupDriver: "systemd" failSwapOn: false volumePluginDir: ` + microshiftDataDir + `/kubelet-plugins/volume/exec clusterDNS: - - ` + cfg.Cluster.DNS + ` + - ` + cfg.Network.DNS + ` clusterDomain: cluster.local containerLogMaxSize: 50Mi maxPods: 250 From 43510fa95db94aad1f88da8ade00c306499d0b64 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 18:36:13 -0500 Subject: [PATCH 07/17] USHIFT-936: merge MicroshiftConfig into Config --- etcd/cmd/microshift-etcd/run.go | 4 +- .../openshift/microshift/pkg/config/config.go | 68 ++++++++----------- pkg/assets/crd.go | 4 +- pkg/cmd/init.go | 6 +- pkg/cmd/run.go | 2 +- pkg/cmd/showConfig.go | 4 +- pkg/components/components.go | 2 +- pkg/components/controllers.go | 6 +- pkg/components/networking.go | 2 +- pkg/components/render.go | 2 +- pkg/components/storage.go | 2 +- pkg/config/config.go | 68 ++++++++----------- pkg/config/config_test.go | 16 ++--- pkg/controllers/cluster-policy-controller.go | 4 +- pkg/controllers/etcd.go | 4 +- pkg/controllers/infra-services-controller.go | 6 +- pkg/controllers/kube-apiserver.go | 6 +- pkg/controllers/kube-controller-manager.go | 4 +- pkg/controllers/kube-scheduler.go | 6 +- pkg/controllers/openshift-crd-manager.go | 4 +- .../openshift-default-scc-manager.go | 6 +- .../openshift-route-controller-manager.go | 6 +- pkg/controllers/version.go | 4 +- pkg/kustomize/apply.go | 2 +- pkg/loadbalancerservice/controller.go | 2 +- pkg/mdns/controller.go | 2 +- pkg/node/kubelet.go | 6 +- pkg/node/netconfig.go | 4 +- pkg/sysconfwatch/sysconfwatch_linux.go | 2 +- 29 files changed, 112 insertions(+), 142 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index 4806b24323..2a61c8b408 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -55,7 +55,7 @@ type EtcdService struct { doStartupDefrag bool } -func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { +func NewEtcd(cfg *config.Config) *EtcdService { s := &EtcdService{} s.configure(cfg) return s @@ -63,7 +63,7 @@ func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { func (s *EtcdService) Name() string { return "etcd" } -func (s *EtcdService) configure(cfg *config.MicroshiftConfig) { +func (s *EtcdService) configure(cfg *config.Config) { s.minDefragBytes = cfg.Etcd.MinDefragBytes s.maxFragmentedPercentage = cfg.Etcd.MaxFragmentedPercentage s.defragCheckFreq = cfg.Etcd.DefragCheckFreq diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 3fe7145e12..7e70bc97e9 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -51,7 +51,7 @@ type IngressConfig struct { type EtcdConfig struct { // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimit uint64 `json:"memoryLimit"` + MemoryLimitMB uint64 `json:"memoryLimitMB"` // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value QuotaBackendBytes int64 `json:"-"` @@ -67,35 +67,21 @@ type EtcdConfig struct { DoStartupDefrag bool `json:"-"` } -type MicroshiftConfig struct { - Ingress IngressConfig `json:"-"` - Etcd EtcdConfig `json:"etcd"` - - DNS DNS `json:"-"` - Node Node `json:"-"` - Debugging Debugging `json:"debugging"` - ApiServer ApiServer `json:"-"` - Network Network `json:"-"` -} - -// Top level config file -type Config struct { - DNS DNS `json:"dns"` - Network Network `json:"network"` - Node Node `json:"node"` - ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` - Etcd Etcd `json:"etcd"` -} - const ( // Etcd performance degrades significantly if the memory available is less than 50MB, enfore this minimum. EtcdMinimumMemoryLimit = 50 ) -type Etcd struct { - // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimitMB uint64 `json:"memoryLimitMB"` +type Config struct { + DNS DNS `json:"dns"` + Network Network `json:"network"` + Node Node `json:"node"` + ApiServer ApiServer `json:"apiServer"` + Etcd EtcdConfig `json:"etcd"` + Debugging Debugging `json:"debugging"` + + // Internal-only fields + Ingress IngressConfig `json:"-"` } type Network struct { @@ -196,11 +182,11 @@ const ( ) // KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *MicroshiftConfig) KubeConfigPath(id KubeConfigID) string { +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { return filepath.Join(dataDir, "resources", string(id), "kubeconfig") } -func (cfg *MicroshiftConfig) KubeConfigAdminPath(id string) string { +func (cfg *Config) KubeConfigAdminPath(id string) string { return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") } @@ -226,7 +212,7 @@ func getAllHostnames() ([]string, error) { return allHostnames, nil } -func NewMicroshiftConfig() *MicroshiftConfig { +func NewMicroshiftConfig() *Config { nodeName, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -240,7 +226,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { klog.Fatalf("failed to get all hostnames: %v", err) } - return &MicroshiftConfig{ + return &Config{ Debugging: Debugging{ LogLevel: "Normal", }, @@ -268,7 +254,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { DNS: "10.43.0.10", }, Etcd: EtcdConfig{ - MemoryLimit: 0, // No limit + MemoryLimitMB: 0, // No limit MinDefragBytes: 100 * 1024 * 1024, // 100MB MaxFragmentedPercentage: 45, // percent DefragCheckFreq: 5 * time.Minute, @@ -279,7 +265,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { } // Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *MicroshiftConfig) isDefaultNodeName() bool { +func (c *Config) isDefaultNodeName() bool { hostname, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -288,7 +274,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { } // Read or set the NodeName that will be used for this MicroShift instance -func (c *MicroshiftConfig) establishNodeName() (string, error) { +func (c *Config) establishNodeName() (string, error) { filePath := filepath.Join(GetDataDir(), ".nodename") contents, err := os.ReadFile(filePath) if os.IsNotExist(err) { @@ -305,7 +291,7 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { } // Validate the NodeName to be used for this MicroShift instances -func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { +func (c *Config) validateNodeName(isDefaultNodeName bool) error { if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } @@ -330,7 +316,7 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *MicroshiftConfig) ApiServerPort() (int, error) { +func (c *Config) ApiServerPort() (int, error) { var port string parsed, err := url.Parse(c.ApiServer.URL) @@ -395,7 +381,7 @@ func StringInList(s string, list []string) bool { return false } -func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { +func (c *Config) ReadFromConfigFile(configFile string) error { contents, err := os.ReadFile(configFile) if err != nil { return fmt.Errorf("reading config file %q: %v", configFile, err) @@ -405,7 +391,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return fmt.Errorf("decoding config file %s: %v", configFile, err) } - // Wire new Config type to existing MicroshiftConfig + // Wire new Config type to existing Config c.Node = config.Node c.Debugging = config.Debugging c.Network = config.Network @@ -420,16 +406,16 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { if config.Etcd.MemoryLimitMB > 0 { // If the memory limit is than the minimum, set it to the minimum and continue. if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { - c.Etcd.MemoryLimit = EtcdMinimumMemoryLimit + c.Etcd.MemoryLimitMB = EtcdMinimumMemoryLimit } else { - c.Etcd.MemoryLimit = config.Etcd.MemoryLimitMB + c.Etcd.MemoryLimitMB = config.Etcd.MemoryLimitMB } } return nil } -func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { +func (c *Config) computeAndUpdateClusterDNS() error { if len(c.Network.ServiceNetwork) == 0 { return fmt.Errorf("network.serviceNetwork not filled in") } @@ -444,7 +430,7 @@ func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { // Note: add a configFile parameter here because of unit test requiring custom // local directory -func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { +func (c *Config) ReadAndValidate(configFile string) error { if configFile != "" { if err := c.ReadFromConfigFile(configFile); err != nil { return err @@ -559,7 +545,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *MicroshiftConfig) GetVerbosity() int { +func (c *Config) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/assets/crd.go b/pkg/assets/crd.go index 392e2b9940..077f5f5507 100644 --- a/pkg/assets/crd.go +++ b/pkg/assets/crd.go @@ -70,7 +70,7 @@ func isEstablished(cs *apiextclientv1.ApiextensionsV1Client, obj apiruntime.Obje return false, err } -func WaitForCrdsEstablished(cfg *config.MicroshiftConfig) error { +func WaitForCrdsEstablished(cfg *config.Config) error { restConfig, err := clientcmd.BuildConfigFromFlags("", cfg.KubeConfigPath(config.KubeAdmin)) if err != nil { return err @@ -115,7 +115,7 @@ func applyCRD(client *apiextclientv1.ApiextensionsV1Client, crd *apiextv1.Custom return err } -func ApplyCRDs(cfg *config.MicroshiftConfig) error { +func ApplyCRDs(cfg *config.Config) error { lock.Lock() defer lock.Unlock() diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 3023036d8b..b7a1df47b3 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -36,7 +36,7 @@ import ( var microshiftDataDir = config.GetDataDir() -func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { +func initCerts(cfg *config.Config) (*certchains.CertificateChains, error) { certChains, err := certSetup(cfg) if err != nil { return nil, err @@ -59,7 +59,7 @@ func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err return certChains, err } -func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { +func certSetup(cfg *config.Config) (*certchains.CertificateChains, error) { _, svcNet, err := net.ParseCIDR(cfg.Network.ServiceNetwork[0]) if err != nil { return nil, err @@ -366,7 +366,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } func initKubeconfigs( - cfg *config.MicroshiftConfig, + cfg *config.Config, certChains *certchains.CertificateChains, ) error { externalTrustPEM, err := os.ReadFile(cryptomaterial.CACertPath(cryptomaterial.KubeAPIServerExternalSigner(cryptomaterial.CertsDirectory(microshiftDataDir)))) diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 5ae200ba0d..fe71074c7c 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -41,7 +41,7 @@ func NewRunMicroshiftCommand() *cobra.Command { return cmd } -func RunMicroshift(cfg *config.MicroshiftConfig) error { +func RunMicroshift(cfg *config.Config) error { if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { klog.Fatalf("Error in reading or validating configuration: %v", err) } diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 8ea011ec65..1ec4bbbd96 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -54,9 +54,7 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command Node: cfg.Node, ApiServer: cfg.ApiServer, Debugging: cfg.Debugging, - Etcd: config.Etcd{ - MemoryLimitMB: cfg.Etcd.MemoryLimit, - }, + Etcd: cfg.Etcd, } marshalled, err := yaml.Marshal(userCfg) cmdutil.CheckErr(err) diff --git a/pkg/components/components.go b/pkg/components/components.go index c20b5f2fc9..aaadbca36f 100755 --- a/pkg/components/components.go +++ b/pkg/components/components.go @@ -7,7 +7,7 @@ import ( var microshiftDataDir = config.GetDataDir() -func StartComponents(cfg *config.MicroshiftConfig) error { +func StartComponents(cfg *config.Config) error { kubeAdminConfig := cfg.KubeConfigPath(config.KubeAdmin) if err := startServiceCAController(cfg, kubeAdminConfig); err != nil { diff --git a/pkg/components/controllers.go b/pkg/components/controllers.go index b69f965cf7..13c4038980 100644 --- a/pkg/components/controllers.go +++ b/pkg/components/controllers.go @@ -9,7 +9,7 @@ import ( "k8s.io/klog/v2" ) -func startServiceCAController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startServiceCAController(cfg *config.Config, kubeconfigPath string) error { var ( //TODO: fix the rolebinding and sa clusterRoleBinding = []string{ @@ -101,7 +101,7 @@ func startServiceCAController(cfg *config.MicroshiftConfig, kubeconfigPath strin return nil } -func startIngressController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startIngressController(cfg *config.Config, kubeconfigPath string) error { var ( clusterRoleBinding = []string{ "components/openshift-router/cluster-role-binding.yaml", @@ -178,7 +178,7 @@ func startIngressController(cfg *config.MicroshiftConfig, kubeconfigPath string) return nil } -func startDNSController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startDNSController(cfg *config.Config, kubeconfigPath string) error { var ( clusterRoleBinding = []string{ "components/openshift-dns/dns/cluster-role-binding.yaml", diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 0cefcd2e46..d0c75b2435 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -10,7 +10,7 @@ import ( "k8s.io/klog/v2" ) -func startCNIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startCNIPlugin(cfg *config.Config, kubeconfigPath string) error { var ( ns = []string{ "components/ovn/namespace.yaml", diff --git a/pkg/components/render.go b/pkg/components/render.go index b2f5b0dbce..5511218cf8 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -20,7 +20,7 @@ var templateFuncs = map[string]interface{}{ "Sha256sum": func(s string) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(s))) }, } -func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderParams) assets.RenderParams { +func renderParamsFromConfig(cfg *config.Config, extra assets.RenderParams) assets.RenderParams { params := map[string]interface{}{ "ReleaseImage": release.Image, "NodeName": cfg.Node.HostnameOverride, diff --git a/pkg/components/storage.go b/pkg/components/storage.go index 1d13590b1d..b4bd0b7531 100644 --- a/pkg/components/storage.go +++ b/pkg/components/storage.go @@ -24,7 +24,7 @@ func getCSIPluginConfig() (*lvmd.Lvmd, error) { return lvmd.DefaultLvmdConfig() } -func startCSIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startCSIPlugin(cfg *config.Config, kubeconfigPath string) error { var ( ns = []string{ "components/lvms/topolvm-openshift-storage_namespace.yaml", diff --git a/pkg/config/config.go b/pkg/config/config.go index 3fe7145e12..7e70bc97e9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -51,7 +51,7 @@ type IngressConfig struct { type EtcdConfig struct { // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimit uint64 `json:"memoryLimit"` + MemoryLimitMB uint64 `json:"memoryLimitMB"` // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value QuotaBackendBytes int64 `json:"-"` @@ -67,35 +67,21 @@ type EtcdConfig struct { DoStartupDefrag bool `json:"-"` } -type MicroshiftConfig struct { - Ingress IngressConfig `json:"-"` - Etcd EtcdConfig `json:"etcd"` - - DNS DNS `json:"-"` - Node Node `json:"-"` - Debugging Debugging `json:"debugging"` - ApiServer ApiServer `json:"-"` - Network Network `json:"-"` -} - -// Top level config file -type Config struct { - DNS DNS `json:"dns"` - Network Network `json:"network"` - Node Node `json:"node"` - ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` - Etcd Etcd `json:"etcd"` -} - const ( // Etcd performance degrades significantly if the memory available is less than 50MB, enfore this minimum. EtcdMinimumMemoryLimit = 50 ) -type Etcd struct { - // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimitMB uint64 `json:"memoryLimitMB"` +type Config struct { + DNS DNS `json:"dns"` + Network Network `json:"network"` + Node Node `json:"node"` + ApiServer ApiServer `json:"apiServer"` + Etcd EtcdConfig `json:"etcd"` + Debugging Debugging `json:"debugging"` + + // Internal-only fields + Ingress IngressConfig `json:"-"` } type Network struct { @@ -196,11 +182,11 @@ const ( ) // KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *MicroshiftConfig) KubeConfigPath(id KubeConfigID) string { +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { return filepath.Join(dataDir, "resources", string(id), "kubeconfig") } -func (cfg *MicroshiftConfig) KubeConfigAdminPath(id string) string { +func (cfg *Config) KubeConfigAdminPath(id string) string { return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") } @@ -226,7 +212,7 @@ func getAllHostnames() ([]string, error) { return allHostnames, nil } -func NewMicroshiftConfig() *MicroshiftConfig { +func NewMicroshiftConfig() *Config { nodeName, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -240,7 +226,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { klog.Fatalf("failed to get all hostnames: %v", err) } - return &MicroshiftConfig{ + return &Config{ Debugging: Debugging{ LogLevel: "Normal", }, @@ -268,7 +254,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { DNS: "10.43.0.10", }, Etcd: EtcdConfig{ - MemoryLimit: 0, // No limit + MemoryLimitMB: 0, // No limit MinDefragBytes: 100 * 1024 * 1024, // 100MB MaxFragmentedPercentage: 45, // percent DefragCheckFreq: 5 * time.Minute, @@ -279,7 +265,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { } // Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *MicroshiftConfig) isDefaultNodeName() bool { +func (c *Config) isDefaultNodeName() bool { hostname, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -288,7 +274,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { } // Read or set the NodeName that will be used for this MicroShift instance -func (c *MicroshiftConfig) establishNodeName() (string, error) { +func (c *Config) establishNodeName() (string, error) { filePath := filepath.Join(GetDataDir(), ".nodename") contents, err := os.ReadFile(filePath) if os.IsNotExist(err) { @@ -305,7 +291,7 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { } // Validate the NodeName to be used for this MicroShift instances -func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { +func (c *Config) validateNodeName(isDefaultNodeName bool) error { if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } @@ -330,7 +316,7 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *MicroshiftConfig) ApiServerPort() (int, error) { +func (c *Config) ApiServerPort() (int, error) { var port string parsed, err := url.Parse(c.ApiServer.URL) @@ -395,7 +381,7 @@ func StringInList(s string, list []string) bool { return false } -func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { +func (c *Config) ReadFromConfigFile(configFile string) error { contents, err := os.ReadFile(configFile) if err != nil { return fmt.Errorf("reading config file %q: %v", configFile, err) @@ -405,7 +391,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return fmt.Errorf("decoding config file %s: %v", configFile, err) } - // Wire new Config type to existing MicroshiftConfig + // Wire new Config type to existing Config c.Node = config.Node c.Debugging = config.Debugging c.Network = config.Network @@ -420,16 +406,16 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { if config.Etcd.MemoryLimitMB > 0 { // If the memory limit is than the minimum, set it to the minimum and continue. if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { - c.Etcd.MemoryLimit = EtcdMinimumMemoryLimit + c.Etcd.MemoryLimitMB = EtcdMinimumMemoryLimit } else { - c.Etcd.MemoryLimit = config.Etcd.MemoryLimitMB + c.Etcd.MemoryLimitMB = config.Etcd.MemoryLimitMB } } return nil } -func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { +func (c *Config) computeAndUpdateClusterDNS() error { if len(c.Network.ServiceNetwork) == 0 { return fmt.Errorf("network.serviceNetwork not filled in") } @@ -444,7 +430,7 @@ func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { // Note: add a configFile parameter here because of unit test requiring custom // local directory -func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { +func (c *Config) ReadAndValidate(configFile string) error { if configFile != "" { if err := c.ReadFromConfigFile(configFile); err != nil { return err @@ -559,7 +545,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *MicroshiftConfig) GetVerbosity() int { +func (c *Config) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index b91f9afdad..6b98e908be 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -30,7 +30,7 @@ func setupSuiteDataDir(t *testing.T) func() { func TestConfigFile(t *testing.T) { var ttests = []struct { config Config - expected MicroshiftConfig + expected Config expectErr bool }{ { @@ -59,7 +59,7 @@ func TestConfigFile(t *testing.T) { LogLevel: "Debug", }, }, - expected: MicroshiftConfig{ + expected: Config{ Debugging: Debugging{ LogLevel: "Debug", }, @@ -86,7 +86,7 @@ func TestConfigFile(t *testing.T) { DNS: "40.30.0.10", }, Etcd: EtcdConfig{ - MemoryLimit: 0, + MemoryLimitMB: 0, QuotaBackendBytes: 8 * 1024 * 1024 * 1024, MinDefragBytes: 100 * 1024 * 1024, MaxFragmentedPercentage: 45, @@ -136,7 +136,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { var ttests = []struct { name string config Config - expected MicroshiftConfig + expected Config expectErr bool }{ { @@ -166,7 +166,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { LogLevel: "Debug", }, }, - expected: MicroshiftConfig{ + expected: Config{ Debugging: Debugging{ LogLevel: "Debug", }, @@ -194,7 +194,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { DNS: "40.30.0.10", }, Etcd: EtcdConfig{ - MemoryLimit: 0, + MemoryLimitMB: 0, QuotaBackendBytes: 8 * 1024 * 1024 * 1024, MinDefragBytes: 100 * 1024 * 1024, MaxFragmentedPercentage: 45, @@ -211,7 +211,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { SubjectAltNames: []string{"127.0.0.1", "localhost"}, }, }, - expected: MicroshiftConfig{}, + expected: Config{}, expectErr: true, }, { @@ -221,7 +221,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { SubjectAltNames: []string{"kubernetes"}, }, }, - expected: MicroshiftConfig{}, + expected: Config{}, expectErr: true, }, } diff --git a/pkg/controllers/cluster-policy-controller.go b/pkg/controllers/cluster-policy-controller.go index d94fc54de0..45ae5e9f7f 100644 --- a/pkg/controllers/cluster-policy-controller.go +++ b/pkg/controllers/cluster-policy-controller.go @@ -35,7 +35,7 @@ type ClusterPolicyController struct { configErr error } -func NewClusterPolicyController(cfg *config.MicroshiftConfig) *ClusterPolicyController { +func NewClusterPolicyController(cfg *config.Config) *ClusterPolicyController { s := &ClusterPolicyController{} s.configErr = s.configure(cfg) return s @@ -44,7 +44,7 @@ func NewClusterPolicyController(cfg *config.MicroshiftConfig) *ClusterPolicyCont func (s *ClusterPolicyController) Name() string { return "cluster-policy-controller" } func (s *ClusterPolicyController) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *ClusterPolicyController) configure(cfg *config.MicroshiftConfig) error { +func (s *ClusterPolicyController) configure(cfg *config.Config) error { s.kubeconfig = cfg.KubeConfigPath(config.ClusterPolicyController) scheme := runtime.NewScheme() diff --git a/pkg/controllers/etcd.go b/pkg/controllers/etcd.go index 98e56458ae..dee64858dd 100644 --- a/pkg/controllers/etcd.go +++ b/pkg/controllers/etcd.go @@ -40,9 +40,9 @@ type EtcdService struct { memoryLimit uint64 } -func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { +func NewEtcd(cfg *config.Config) *EtcdService { return &EtcdService{ - memoryLimit: cfg.Etcd.MemoryLimit, + memoryLimit: cfg.Etcd.MemoryLimitMB, } } diff --git a/pkg/controllers/infra-services-controller.go b/pkg/controllers/infra-services-controller.go index 08a5c85298..7c5f8af766 100644 --- a/pkg/controllers/infra-services-controller.go +++ b/pkg/controllers/infra-services-controller.go @@ -26,10 +26,10 @@ import ( ) type InfrastructureServicesManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewInfrastructureServices(cfg *config.MicroshiftConfig) *InfrastructureServicesManager { +func NewInfrastructureServices(cfg *config.Config) *InfrastructureServicesManager { s := &InfrastructureServicesManager{} s.cfg = cfg return s @@ -62,7 +62,7 @@ func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- st return ctx.Err() } -func applyDefaultRBACs(cfg *config.MicroshiftConfig) error { +func applyDefaultRBACs(cfg *config.Config) error { kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) var ( cr = []string{ diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index 88ec80b3b3..d9b38fbe3f 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -80,7 +80,7 @@ type KubeAPIServer struct { advertiseAddress string } -func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer { +func NewKubeAPIServer(cfg *config.Config) *KubeAPIServer { s := &KubeAPIServer{} if err := s.configure(cfg); err != nil { s.configureErr = err @@ -91,7 +91,7 @@ func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer { func (s *KubeAPIServer) Name() string { return "kube-apiserver" } func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd", "network-configuration"} } -func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { +func (s *KubeAPIServer) configure(cfg *config.Config) error { s.verbosity = cfg.GetVerbosity() certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) @@ -255,7 +255,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { return nil } -func (s *KubeAPIServer) configureAuditPolicy(cfg *config.MicroshiftConfig) error { +func (s *KubeAPIServer) configureAuditPolicy(cfg *config.Config) error { data := []byte(` apiVersion: audit.k8s.io/v1 kind: Policy diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 03e9ff30d6..3a2126cb58 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -49,7 +49,7 @@ type KubeControllerManager struct { configureErr error } -func NewKubeControllerManager(cfg *config.MicroshiftConfig) *KubeControllerManager { +func NewKubeControllerManager(cfg *config.Config) *KubeControllerManager { s := &KubeControllerManager{} // TODO: manage and invoke the configure bits independently outside of this. s.args, s.applyFn, s.configureErr = configure(cfg) @@ -74,7 +74,7 @@ func kcmServiceAccountPrivateKeyFile() string { return microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.key" } -func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() error, err error) { +func configure(cfg *config.Config) (args []string, applyFn func() error, err error) { kubeConfig := cfg.KubeConfigPath(config.KubeControllerManager) clusterSigningKey, clusterSigningCert := kcmClusterSigningCertKeyAndFile() diff --git a/pkg/controllers/kube-scheduler.go b/pkg/controllers/kube-scheduler.go index 78b219a259..d1cd500799 100644 --- a/pkg/controllers/kube-scheduler.go +++ b/pkg/controllers/kube-scheduler.go @@ -39,7 +39,7 @@ type KubeScheduler struct { kubeconfig string } -func NewKubeScheduler(cfg *config.MicroshiftConfig) *KubeScheduler { +func NewKubeScheduler(cfg *config.Config) *KubeScheduler { s := &KubeScheduler{} s.configure(cfg) return s @@ -48,7 +48,7 @@ func NewKubeScheduler(cfg *config.MicroshiftConfig) *KubeScheduler { func (s *KubeScheduler) Name() string { return "kube-scheduler" } func (s *KubeScheduler) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *KubeScheduler) configure(cfg *config.MicroshiftConfig) { +func (s *KubeScheduler) configure(cfg *config.Config) { if err := s.writeConfig(cfg); err != nil { klog.Fatalf("failed to write kube-scheduler config: %v", err) } @@ -60,7 +60,7 @@ func (s *KubeScheduler) configure(cfg *config.MicroshiftConfig) { s.kubeconfig = cfg.KubeConfigPath(config.KubeScheduler) } -func (s *KubeScheduler) writeConfig(cfg *config.MicroshiftConfig) error { +func (s *KubeScheduler) writeConfig(cfg *config.Config) error { data := []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3 kind: KubeSchedulerConfiguration clientConnection: diff --git a/pkg/controllers/openshift-crd-manager.go b/pkg/controllers/openshift-crd-manager.go index f505b650d0..de328afc89 100644 --- a/pkg/controllers/openshift-crd-manager.go +++ b/pkg/controllers/openshift-crd-manager.go @@ -24,10 +24,10 @@ import ( ) type OpenShiftCRDManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewOpenShiftCRDManager(cfg *config.MicroshiftConfig) *OpenShiftCRDManager { +func NewOpenShiftCRDManager(cfg *config.Config) *OpenShiftCRDManager { s := &OpenShiftCRDManager{} s.cfg = cfg return s diff --git a/pkg/controllers/openshift-default-scc-manager.go b/pkg/controllers/openshift-default-scc-manager.go index d933f2015a..19615724c1 100644 --- a/pkg/controllers/openshift-default-scc-manager.go +++ b/pkg/controllers/openshift-default-scc-manager.go @@ -24,10 +24,10 @@ import ( ) type OpenShiftDefaultSCCManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewOpenShiftDefaultSCCManager(cfg *config.MicroshiftConfig) *OpenShiftDefaultSCCManager { +func NewOpenShiftDefaultSCCManager(cfg *config.Config) *OpenShiftDefaultSCCManager { s := &OpenShiftDefaultSCCManager{} s.cfg = cfg return s @@ -51,7 +51,7 @@ func (s *OpenShiftDefaultSCCManager) Run(ctx context.Context, ready chan<- struc return ctx.Err() } -func ApplyDefaultSCCs(cfg *config.MicroshiftConfig) error { +func ApplyDefaultSCCs(cfg *config.Config) error { kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) var ( clusterRole = []string{ diff --git a/pkg/controllers/openshift-route-controller-manager.go b/pkg/controllers/openshift-route-controller-manager.go index bf7df109f0..3bfa301fab 100644 --- a/pkg/controllers/openshift-route-controller-manager.go +++ b/pkg/controllers/openshift-route-controller-manager.go @@ -44,7 +44,7 @@ const ( componentRCM = "route-controller-manager" ) -func NewRouteControllerManager(cfg *config.MicroshiftConfig) *OCPRouteControllerManager { +func NewRouteControllerManager(cfg *config.Config) *OCPRouteControllerManager { s := &OCPRouteControllerManager{} s.configure(cfg) return s @@ -55,13 +55,13 @@ func (s *OCPRouteControllerManager) Dependencies() []string { return []string{"kube-apiserver", "openshift-crd-manager"} } -func (s *OCPRouteControllerManager) configure(cfg *config.MicroshiftConfig) { +func (s *OCPRouteControllerManager) configure(cfg *config.Config) { s.kubeconfig = cfg.KubeConfigPath(config.RouteControllerManager) s.kubeadmconfig = cfg.KubeConfigPath(config.KubeAdmin) s.config = s.writeConfig(cfg) } -func (s *OCPRouteControllerManager) writeConfig(cfg *config.MicroshiftConfig) *openshiftcontrolplanev1.OpenShiftControllerManagerConfig { +func (s *OCPRouteControllerManager) writeConfig(cfg *config.Config) *openshiftcontrolplanev1.OpenShiftControllerManagerConfig { servingCertDir := cryptomaterial.RouteControllerManagerServingCertDir(cryptomaterial.CertsDirectory(microshiftDataDir)) c := &openshiftcontrolplanev1.OpenShiftControllerManagerConfig{ diff --git a/pkg/controllers/version.go b/pkg/controllers/version.go index adb7e04c58..4495cbcc8f 100644 --- a/pkg/controllers/version.go +++ b/pkg/controllers/version.go @@ -25,10 +25,10 @@ import ( ) type VersionManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewVersionManager(cfg *config.MicroshiftConfig) *VersionManager { +func NewVersionManager(cfg *config.Config) *VersionManager { s := &VersionManager{} s.cfg = cfg return s diff --git a/pkg/kustomize/apply.go b/pkg/kustomize/apply.go index bfe06de1c6..c87453bd86 100644 --- a/pkg/kustomize/apply.go +++ b/pkg/kustomize/apply.go @@ -30,7 +30,7 @@ type Kustomizer struct { kubeconfig string } -func NewKustomizer(cfg *config.MicroshiftConfig) *Kustomizer { +func NewKustomizer(cfg *config.Config) *Kustomizer { return &Kustomizer{ paths: microshiftManifestsDir, kubeconfig: cfg.KubeConfigPath(config.KubeAdmin), diff --git a/pkg/loadbalancerservice/controller.go b/pkg/loadbalancerservice/controller.go index 2a5d9a0ad3..3b94100ef6 100644 --- a/pkg/loadbalancerservice/controller.go +++ b/pkg/loadbalancerservice/controller.go @@ -34,7 +34,7 @@ type LoadbalancerServiceController struct { var _ servicemanager.Service = &LoadbalancerServiceController{} -func NewLoadbalancerServiceController(cfg *config.MicroshiftConfig) *LoadbalancerServiceController { +func NewLoadbalancerServiceController(cfg *config.Config) *LoadbalancerServiceController { return &LoadbalancerServiceController{ NodeIP: cfg.Node.NodeIP, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), diff --git a/pkg/mdns/controller.go b/pkg/mdns/controller.go index 3daee405c2..29d7231aaf 100644 --- a/pkg/mdns/controller.go +++ b/pkg/mdns/controller.go @@ -23,7 +23,7 @@ type MicroShiftmDNSController struct { stopCh chan struct{} } -func NewMicroShiftmDNSController(cfg *config.MicroshiftConfig) *MicroShiftmDNSController { +func NewMicroShiftmDNSController(cfg *config.Config) *MicroShiftmDNSController { return &MicroShiftmDNSController{ NodeIP: cfg.Node.NodeIP, NodeName: cfg.Node.HostnameOverride, diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index 1192c1f784..55ea2b1695 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -51,7 +51,7 @@ type KubeletServer struct { kubeconfig *kubeletconfig.KubeletConfiguration } -func NewKubeletServer(cfg *config.MicroshiftConfig) *KubeletServer { +func NewKubeletServer(cfg *config.Config) *KubeletServer { s := &KubeletServer{} s.configure(cfg) return s @@ -60,7 +60,7 @@ func NewKubeletServer(cfg *config.MicroshiftConfig) *KubeletServer { func (s *KubeletServer) Name() string { return componentKubelet } func (s *KubeletServer) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { +func (s *KubeletServer) configure(cfg *config.Config) { if err := s.writeConfig(cfg); err != nil { klog.Fatalf("Failed to write kubelet config", err) @@ -93,7 +93,7 @@ func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { s.kubeletflags = kubeletFlags } -func (s *KubeletServer) writeConfig(cfg *config.MicroshiftConfig) error { +func (s *KubeletServer) writeConfig(cfg *config.Config) error { certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) servingCertDir := cryptomaterial.KubeletServingCertDir(certsDir) diff --git a/pkg/node/netconfig.go b/pkg/node/netconfig.go index e92f9c81d3..a202bc5a04 100644 --- a/pkg/node/netconfig.go +++ b/pkg/node/netconfig.go @@ -38,7 +38,7 @@ type NetworkConfiguration struct { skipInterfaceConfiguration bool } -func NewNetworkConfiguration(cfg *config.MicroshiftConfig) *NetworkConfiguration { +func NewNetworkConfiguration(cfg *config.Config) *NetworkConfiguration { n := &NetworkConfiguration{} n.configure(cfg) return n @@ -47,7 +47,7 @@ func NewNetworkConfiguration(cfg *config.MicroshiftConfig) *NetworkConfiguration func (n *NetworkConfiguration) Name() string { return componentNetworkConfiguration } func (n *NetworkConfiguration) Dependencies() []string { return []string{} } -func (n *NetworkConfiguration) configure(cfg *config.MicroshiftConfig) { +func (n *NetworkConfiguration) configure(cfg *config.Config) { n.kasAdvertiseAddress = cfg.ApiServer.AdvertiseAddress n.skipInterfaceConfiguration = cfg.ApiServer.SkipInterface } diff --git a/pkg/sysconfwatch/sysconfwatch_linux.go b/pkg/sysconfwatch/sysconfwatch_linux.go index 0c18891ff7..02f43e7018 100644 --- a/pkg/sysconfwatch/sysconfwatch_linux.go +++ b/pkg/sysconfwatch/sysconfwatch_linux.go @@ -36,7 +36,7 @@ type SysConfWatchController struct { timerFd int } -func NewSysConfWatchController(cfg *config.MicroshiftConfig) *SysConfWatchController { +func NewSysConfWatchController(cfg *config.Config) *SysConfWatchController { // Create a realtime clock timer with asynchronous read support fd, err := unix.TimerfdCreate(unix.CLOCK_REALTIME, unix.TFD_CLOEXEC|unix.TFD_NONBLOCK) if err != nil { From cffbb13d0ce89933e7b19da8dd7833ccd00868bb Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 4 Mar 2023 11:49:32 -0500 Subject: [PATCH 08/17] USHIFT-936: split up large config.go Move data structures and related functions into their own files. --- .../microshift/pkg/config/apiserver.go | 44 ++ .../openshift/microshift/pkg/config/config.go | 474 ------------------ .../microshift/pkg/config/debugging.go | 25 + .../openshift/microshift/pkg/config/dns.go | 14 + .../openshift/microshift/pkg/config/etcd.go | 33 ++ .../openshift/microshift/pkg/config/files.go | 204 ++++++++ .../openshift/microshift/pkg/config/flags.go | 23 + .../microshift/pkg/config/ingress.go | 6 + .../microshift/pkg/config/kubeconfig.go | 24 + .../microshift/pkg/config/network.go | 63 +++ .../openshift/microshift/pkg/config/node.go | 71 +++ .../openshift/microshift/pkg/config/util.go | 21 + pkg/config/apiserver.go | 44 ++ pkg/config/config.go | 474 ------------------ pkg/config/debugging.go | 25 + pkg/config/debugging_test.go | 48 ++ pkg/config/dns.go | 14 + pkg/config/etcd.go | 33 ++ pkg/config/files.go | 204 ++++++++ pkg/config/flags.go | 23 + pkg/config/ingress.go | 6 + pkg/config/kubeconfig.go | 24 + pkg/config/network.go | 63 +++ pkg/config/node.go | 71 +++ pkg/config/util.go | 21 + 25 files changed, 1104 insertions(+), 948 deletions(-) create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/files.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/network.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/node.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/util.go create mode 100644 pkg/config/apiserver.go create mode 100644 pkg/config/debugging.go create mode 100644 pkg/config/debugging_test.go create mode 100644 pkg/config/dns.go create mode 100644 pkg/config/etcd.go create mode 100644 pkg/config/files.go create mode 100644 pkg/config/flags.go create mode 100644 pkg/config/ingress.go create mode 100644 pkg/config/kubeconfig.go create mode 100644 pkg/config/network.go create mode 100644 pkg/config/node.go create mode 100644 pkg/config/util.go diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go new file mode 100644 index 0000000000..dcb398c7a0 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go @@ -0,0 +1,44 @@ +package config + +import ( + "net/url" + "strconv" +) + +type ApiServer struct { + // SubjectAltNames added to API server certs + SubjectAltNames []string `json:"subjectAltNames"` + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. + AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` + + // The URL of the API server + URL string `json:"-"` +} + +// extract the api server port from the cluster URL +func (c *Config) ApiServerPort() (int, error) { + var port string + + parsed, err := url.Parse(c.ApiServer.URL) + if err != nil { + return 0, err + } + + // default empty URL to port 6443 + port = parsed.Port() + if port == "" { + port = "6443" + } + portNum, err := strconv.Atoi(port) + if err != nil { + return 0, err + } + return portNum, nil +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 7e70bc97e9..376cede347 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -2,76 +2,22 @@ package config import ( "bytes" - "errors" "fmt" - "net" - "net/url" "os" "os/exec" - "path/filepath" - "strconv" "strings" "time" - "github.com/apparentlymart/go-cidr/cidr" - "github.com/mitchellh/go-homedir" - "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/component-base/logs" "k8s.io/klog/v2" - "sigs.k8s.io/yaml" "github.com/openshift/microshift/pkg/util" ) const ( - DefaultUserConfigFile = "~/.microshift/config.yaml" - defaultUserDataDir = "~/.microshift/data" - DefaultGlobalConfigFile = "/etc/microshift/config.yaml" - defaultGlobalDataDir = "/var/lib/microshift" - // for files managed via management system in /etc, i.e. user applications - defaultManifestDirEtc = "/etc/microshift/manifests" - // for files embedded in ostree. i.e. cni/other component customizations - defaultManifestDirLib = "/usr/lib/microshift/manifests" - // default DNS resolve file when systemd-resolved is used DefaultSystemdResolvedFile = "/run/systemd/resolve/resolv.conf" ) -var ( - configFile = findConfigFile() - dataDir = findDataDir() - manifestsDir = findManifestsDir() -) - -type IngressConfig struct { - ServingCertificate []byte - ServingKey []byte -} - -type EtcdConfig struct { - // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimitMB uint64 `json:"memoryLimitMB"` - - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendBytes int64 `json:"-"` - - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragBytes`, do a defrag. - MinDefragBytes int64 `json:"-"` - MaxFragmentedPercentage float64 `json:"-"` - - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq time.Duration `json:"-"` - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool `json:"-"` -} - -const ( - // Etcd performance degrades significantly if the memory available is less than 50MB, enfore this minimum. - EtcdMinimumMemoryLimit = 50 -) - type Config struct { DNS DNS `json:"dns"` Network Network `json:"network"` @@ -84,112 +30,6 @@ type Config struct { Ingress IngressConfig `json:"-"` } -type Network struct { - // IP address pool to use for pod IPs. - // This field is immutable after installation. - ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork,omitempty"` - - // IP address pool for services. - // Currently, we only support a single entry here. - // This field is immutable after installation. - ServiceNetwork []string `json:"serviceNetwork,omitempty"` - - // The port range allowed for Services of type NodePort. - // If not specified, the default of 30000-32767 will be used. - // Such Services without a NodePort specified will have one - // automatically allocated from this range. - // This parameter can be updated after the cluster is - // installed. - // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` - ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` - - // The DNS server to use - DNS string `json:"-"` -} - -type ClusterNetworkEntry struct { - // The complete block for pod IPs. - CIDR string `json:"cidr,omitempty"` -} - -type DNS struct { - // baseDomain is the base domain of the cluster. All managed DNS records will - // be sub-domains of this base. - // - // For example, given the base domain `example.com`, router exposed - // domains will be formed as `*.apps.example.com` by default, - // and API service will have a DNS entry for `api.example.com`, - // as well as "api-int.example.com" for internal k8s API access. - // - // Once set, this field cannot be changed. - BaseDomain string `json:"baseDomain"` -} - -type ApiServer struct { - // SubjectAltNames added to API server certs - SubjectAltNames []string `json:"subjectAltNames"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - AdvertiseAddress string `json:"advertiseAddress,omitempty"` - // Determines if kube-apiserver controller should configure the - // AdvertiseAddress in the loopback interface. Automatically computed. - SkipInterface bool `json:"-"` - - // The URL of the API server - URL string `json:"-"` -} - -type Node struct { - // If non-empty, will use this string to identify the node instead of the hostname - HostnameOverride string `json:"hostnameOverride"` - - // IP address of the node, passed to the kubelet. - // If not specified, kubelet will use the node's default IP address. - NodeIP string `json:"nodeIP"` -} - -type Debugging struct { - // Valid values are: "Normal", "Debug", "Trace", "TraceAll". - // Defaults to "Normal". - LogLevel string `json:"logLevel"` -} - -func GetConfigFile() string { - return configFile -} - -func GetDataDir() string { - return dataDir -} - -func GetManifestsDir() []string { - return manifestsDir -} - -// KubeConfigID identifies the different kubeconfigs managed in the DataDir -type KubeConfigID string - -const ( - KubeAdmin KubeConfigID = "kubeadmin" - KubeControllerManager KubeConfigID = "kube-controller-manager" - KubeScheduler KubeConfigID = "kube-scheduler" - Kubelet KubeConfigID = "kubelet" - ClusterPolicyController KubeConfigID = "cluster-policy-controller" - RouteControllerManager KubeConfigID = "route-controller-manager" -) - -// KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *Config) KubeConfigPath(id KubeConfigID) string { - return filepath.Join(dataDir, "resources", string(id), "kubeconfig") -} - -func (cfg *Config) KubeConfigAdminPath(id string) string { - return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") -} - var allHostnames []string func getAllHostnames() ([]string, error) { @@ -263,317 +103,3 @@ func NewMicroshiftConfig() *Config { }, } } - -// Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *Config) isDefaultNodeName() bool { - hostname, err := os.Hostname() - if err != nil { - klog.Fatalf("Failed to get hostname %v", err) - } - return c.Node.HostnameOverride == strings.ToLower(hostname) -} - -// Read or set the NodeName that will be used for this MicroShift instance -func (c *Config) establishNodeName() (string, error) { - filePath := filepath.Join(GetDataDir(), ".nodename") - contents, err := os.ReadFile(filePath) - if os.IsNotExist(err) { - // ensure that dataDir exists - os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { - return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) - } - return c.Node.HostnameOverride, nil - } else if err != nil { - return "", err - } - return string(contents), nil -} - -// Validate the NodeName to be used for this MicroShift instances -func (c *Config) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) - } - - establishedNodeName, err := c.establishNodeName() - if err != nil { - return fmt.Errorf("failed to establish NodeName: %v", err) - } - - if establishedNodeName != c.Node.HostnameOverride { - if !isDefaultNodeName { - return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.Node.HostnameOverride, establishedNodeName) - } else { - c.Node.HostnameOverride = establishedNodeName - klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) - } - } - - return nil -} - -// extract the api server port from the cluster URL -func (c *Config) ApiServerPort() (int, error) { - var port string - - parsed, err := url.Parse(c.ApiServer.URL) - if err != nil { - return 0, err - } - - // default empty URL to port 6443 - port = parsed.Port() - if port == "" { - port = "6443" - } - portNum, err := strconv.Atoi(port) - if err != nil { - return 0, err - } - return portNum, nil -} - -// Returns the default user config file if that exists, else the default global -// config file, else the empty string. -func findConfigFile() string { - userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) - if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { - if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { - return "" - } else { - return DefaultGlobalConfigFile - } - } else { - return userConfigFile - } -} - -// Returns the default user data dir if it exists or the user is non-root. -// Returns the default global data dir otherwise. -func findDataDir() string { - userDataDir, _ := homedir.Expand(defaultUserDataDir) - if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { - if os.Geteuid() > 0 { - return userDataDir - } else { - return defaultGlobalDataDir - } - } else { - return userDataDir - } -} - -// Returns the default manifests directories -func findManifestsDir() []string { - var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} - return manifestsDir -} - -func StringInList(s string, list []string) bool { - for _, x := range list { - if x == s { - return true - } - } - return false -} - -func (c *Config) ReadFromConfigFile(configFile string) error { - contents, err := os.ReadFile(configFile) - if err != nil { - return fmt.Errorf("reading config file %q: %v", configFile, err) - } - var config Config - if err := yaml.Unmarshal(contents, &config); err != nil { - return fmt.Errorf("decoding config file %s: %v", configFile, err) - } - - // Wire new Config type to existing Config - c.Node = config.Node - c.Debugging = config.Debugging - c.Network = config.Network - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - c.DNS = config.DNS - c.ApiServer = config.ApiServer - c.ApiServer.URL = "https://localhost:6443" - - if config.Etcd.MemoryLimitMB > 0 { - // If the memory limit is than the minimum, set it to the minimum and continue. - if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { - c.Etcd.MemoryLimitMB = EtcdMinimumMemoryLimit - } else { - c.Etcd.MemoryLimitMB = config.Etcd.MemoryLimitMB - } - } - - return nil -} - -func (c *Config) computeAndUpdateClusterDNS() error { - if len(c.Network.ServiceNetwork) == 0 { - return fmt.Errorf("network.serviceNetwork not filled in") - } - - clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) - if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) - } - c.Network.DNS = clusterDNS - return nil -} - -// Note: add a configFile parameter here because of unit test requiring custom -// local directory -func (c *Config) ReadAndValidate(configFile string) error { - if configFile != "" { - if err := c.ReadFromConfigFile(configFile); err != nil { - return err - } - } - - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - // If KAS advertise address is not configured then compute it from the service - // CIDR automatically. - if len(c.ApiServer.AdvertiseAddress) == 0 { - // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) - // Since the KAS advertise address was not provided we will default to the - // next immediate subnet after the service CIDR. This is due to the fact - // that using the actual apiserver service IP as an endpoint slice breaks - // host network pods trying to reach apiserver, as the VIP 10.43.0.1:443 is - // not translated to 10.43.0.1:6443. It remains unchanged and therefore - // connects to the ingress router instead, triggering all sorts of errors. - nextSubnet, exceed := cidr.NextSubnet(svcNet, 32) - if exceed { - return fmt.Errorf("unable to compute next subnet from service CIDR") - } - // First and last are the same because of the /32 netmask. - firstValidIP, _ := cidr.AddressRange(nextSubnet) - c.ApiServer.AdvertiseAddress = firstValidIP.String() - c.ApiServer.SkipInterface = false - } else { - c.ApiServer.SkipInterface = true - } - - if len(c.ApiServer.SubjectAltNames) > 0 { - // Any entry in SubjectAltNames will be included in the external access certificates. - // Any of the hostnames and IPs (except the node IP) listed below conflicts with - // other certificates, such as the service network and localhost access. - // The node IP is a bit special. Apiserver k8s service, which holds a service IP - // gets resolved to the node IP. If we include the node IP in the SAN then we have - // an ambiguity, the same IP matches two different certificates and there are errors - // when trying to reach apiserver from within the cluster using the service IP. - // Apiserver will decide which certificate to return to client hello based on SNI - // (which client-go does not use) or raw IP mappings. As soon as there is a match for - // the node IP it returns that certificate, which is the external access one. This - // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate - // is invalid. - u, err := url.Parse(c.ApiServer.URL) - if err != nil { - return fmt.Errorf("failed to parse cluster URL: %v", err) - } - if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { - return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") - } - } else { - if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { - return fmt.Errorf("subjectAltNames must not contain node IP") - } - if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { - return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) - } - } - - if stringSliceContains( - c.ApiServer.SubjectAltNames, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc.cluster.local", - "openshift", - "openshift.default", - "openshift.default.svc", - "openshift.default.svc.cluster.local", - c.ApiServer.AdvertiseAddress, - ) { - return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") - } - } - // Validate NodeName in config file, node-name should not be changed for an already - // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage - // being orphaned or lost, and other side effects. - if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { - klog.Fatalf("Error in validating node name: %v", err) - } - - return nil -} - -// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork -func getClusterDNS(serviceCIDR string) (string, error) { - _, service, err := net.ParseCIDR(serviceCIDR) - if err != nil { - return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) - } - dnsClusterIP, err := cidr.Host(service, 10) - if err != nil { - return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) - } - - return dnsClusterIP.String(), nil -} - -func stringSliceContains(list []string, elements ...string) bool { - for _, value := range list { - for _, element := range elements { - if value == element { - return true - } - } - } - return false -} - -// GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { - var verbosity int - switch c.Debugging.LogLevel { - case "Normal": - verbosity = 2 - case "Debug": - verbosity = 4 - case "Trace": - verbosity = 6 - case "TraceAll": - verbosity = 8 - default: - verbosity = 2 - } - return verbosity -} - -func HideUnsupportedFlags(flags *pflag.FlagSet) { - // hide logging flags that we do not use/support - loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) - logs.AddFlags(loggingFlags) - - supportedLoggingFlags := sets.NewString("v") - - loggingFlags.VisitAll(func(pf *pflag.Flag) { - if !supportedLoggingFlags.Has(pf.Name) { - flags.MarkHidden(pf.Name) - } - }) - - flags.MarkHidden("version") -} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go new file mode 100644 index 0000000000..a540f5490c --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go @@ -0,0 +1,25 @@ +package config + +type Debugging struct { + // Valid values are: "Normal", "Debug", "Trace", "TraceAll". + // Defaults to "Normal". + LogLevel string `json:"logLevel"` +} + +// GetVerbosity returns the numerical value for LogLevel which is an enum +func (c *Config) GetVerbosity() int { + var verbosity int + switch c.Debugging.LogLevel { + case "Normal": + verbosity = 2 + case "Debug": + verbosity = 4 + case "Trace": + verbosity = 6 + case "TraceAll": + verbosity = 8 + default: + verbosity = 2 + } + return verbosity +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go new file mode 100644 index 0000000000..ca6f11e88e --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go @@ -0,0 +1,14 @@ +package config + +type DNS struct { + // baseDomain is the base domain of the cluster. All managed DNS records will + // be sub-domains of this base. + // + // For example, given the base domain `example.com`, router exposed + // domains will be formed as `*.apps.example.com` by default, + // and API service will have a DNS entry for `api.example.com`, + // as well as "api-int.example.com" for internal k8s API access. + // + // Once set, this field cannot be changed. + BaseDomain string `json:"baseDomain"` +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go new file mode 100644 index 0000000000..dc26b92ec2 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go @@ -0,0 +1,33 @@ +package config + +import "time" + +const ( + // Etcd performance degrades significantly if the memory available + // is less than 50MB, enforce this minimum. + EtcdMinimumMemoryLimit = 50 +) + +type EtcdConfig struct { + // Set a memory limit on the etcd process; etcd will begin paging + // memory when it gets to this value. 0 means no limit. + MemoryLimitMB uint64 `json:"memoryLimitMB"` + + // The limit on the size of the etcd database; etcd will start + // failing writes if its size on disk reaches this value + QuotaBackendBytes int64 `json:"-"` + + // If the backend is fragmented more than + // `maxFragmentedPercentage` and the database size is greater than + // `minDefragBytes`, do a defrag. + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"-"` + + // How often to check the conditions for defragging (0 means no + // defrags, except for a single on startup if `doStartupDefrag` is + // set). + DefragCheckFreq time.Duration `json:"-"` + + // Whether or not to do a defrag when the server finishes starting + DoStartupDefrag bool `json:"-"` +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go new file mode 100644 index 0000000000..7319033cff --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go @@ -0,0 +1,204 @@ +package config + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + + "github.com/apparentlymart/go-cidr/cidr" + "github.com/mitchellh/go-homedir" + "k8s.io/klog/v2" + "sigs.k8s.io/yaml" +) + +const ( + DefaultUserConfigFile = "~/.microshift/config.yaml" + defaultUserDataDir = "~/.microshift/data" + DefaultGlobalConfigFile = "/etc/microshift/config.yaml" + defaultGlobalDataDir = "/var/lib/microshift" + // for files managed via management system in /etc, i.e. user applications + defaultManifestDirEtc = "/etc/microshift/manifests" + // for files embedded in ostree. i.e. cni/other component customizations + defaultManifestDirLib = "/usr/lib/microshift/manifests" +) + +var ( + configFile = findConfigFile() + dataDir = findDataDir() + manifestsDir = findManifestsDir() +) + +func GetConfigFile() string { + return configFile +} + +func GetDataDir() string { + return dataDir +} + +func GetManifestsDir() []string { + return manifestsDir +} + +// Returns the default user config file if that exists, else the default global +// config file, else the empty string. +func findConfigFile() string { + userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) + if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { + if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { + return "" + } else { + return DefaultGlobalConfigFile + } + } else { + return userConfigFile + } +} + +// Returns the default user data dir if it exists or the user is non-root. +// Returns the default global data dir otherwise. +func findDataDir() string { + userDataDir, _ := homedir.Expand(defaultUserDataDir) + if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { + if os.Geteuid() > 0 { + return userDataDir + } else { + return defaultGlobalDataDir + } + } else { + return userDataDir + } +} + +// Returns the default manifests directories +func findManifestsDir() []string { + var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} + return manifestsDir +} + +func (c *Config) ReadFromConfigFile(configFile string) error { + contents, err := os.ReadFile(configFile) + if err != nil { + return fmt.Errorf("reading config file %q: %v", configFile, err) + } + var config Config + if err := yaml.Unmarshal(contents, &config); err != nil { + return fmt.Errorf("decoding config file %s: %v", configFile, err) + } + + // Wire new Config type to existing Config + c.Node = config.Node + c.Debugging = config.Debugging + c.Network = config.Network + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + } + + c.DNS = config.DNS + c.ApiServer = config.ApiServer + c.ApiServer.URL = "https://localhost:6443" + + if config.Etcd.MemoryLimitMB > 0 { + // If the memory limit is than the minimum, set it to the minimum and continue. + if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { + c.Etcd.MemoryLimitMB = EtcdMinimumMemoryLimit + } else { + c.Etcd.MemoryLimitMB = config.Etcd.MemoryLimitMB + } + } + + return nil +} + +// Note: add a configFile parameter here because of unit test requiring custom +// local directory +func (c *Config) ReadAndValidate(configFile string) error { + if configFile != "" { + if err := c.ReadFromConfigFile(configFile); err != nil { + return err + } + } + + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + } + + // If KAS advertise address is not configured then compute it from the service + // CIDR automatically. + if len(c.ApiServer.AdvertiseAddress) == 0 { + // unchecked error because this was done when getting cluster DNS + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) + // Since the KAS advertise address was not provided we will default to the + // next immediate subnet after the service CIDR. This is due to the fact + // that using the actual apiserver service IP as an endpoint slice breaks + // host network pods trying to reach apiserver, as the VIP 10.43.0.1:443 is + // not translated to 10.43.0.1:6443. It remains unchanged and therefore + // connects to the ingress router instead, triggering all sorts of errors. + nextSubnet, exceed := cidr.NextSubnet(svcNet, 32) + if exceed { + return fmt.Errorf("unable to compute next subnet from service CIDR") + } + // First and last are the same because of the /32 netmask. + firstValidIP, _ := cidr.AddressRange(nextSubnet) + c.ApiServer.AdvertiseAddress = firstValidIP.String() + c.ApiServer.SkipInterface = false + } else { + c.ApiServer.SkipInterface = true + } + + if len(c.ApiServer.SubjectAltNames) > 0 { + // Any entry in SubjectAltNames will be included in the external access certificates. + // Any of the hostnames and IPs (except the node IP) listed below conflicts with + // other certificates, such as the service network and localhost access. + // The node IP is a bit special. Apiserver k8s service, which holds a service IP + // gets resolved to the node IP. If we include the node IP in the SAN then we have + // an ambiguity, the same IP matches two different certificates and there are errors + // when trying to reach apiserver from within the cluster using the service IP. + // Apiserver will decide which certificate to return to client hello based on SNI + // (which client-go does not use) or raw IP mappings. As soon as there is a match for + // the node IP it returns that certificate, which is the external access one. This + // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate + // is invalid. + u, err := url.Parse(c.ApiServer.URL) + if err != nil { + return fmt.Errorf("failed to parse cluster URL: %v", err) + } + if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { + return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") + } + } else { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { + return fmt.Errorf("subjectAltNames must not contain node IP") + } + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) + } + } + + if stringSliceContains( + c.ApiServer.SubjectAltNames, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + "openshift", + "openshift.default", + "openshift.default.svc", + "openshift.default.svc.cluster.local", + c.ApiServer.AdvertiseAddress, + ) { + return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") + } + } + // Validate NodeName in config file, node-name should not be changed for an already + // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage + // being orphaned or lost, and other side effects. + if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { + klog.Fatalf("Error in validating node name: %v", err) + } + + return nil +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go new file mode 100644 index 0000000000..e15ee75a1c --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go @@ -0,0 +1,23 @@ +package config + +import ( + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/component-base/logs" +) + +func HideUnsupportedFlags(flags *pflag.FlagSet) { + // hide logging flags that we do not use/support + loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) + logs.AddFlags(loggingFlags) + + supportedLoggingFlags := sets.NewString("v") + + loggingFlags.VisitAll(func(pf *pflag.Flag) { + if !supportedLoggingFlags.Has(pf.Name) { + flags.MarkHidden(pf.Name) + } + }) + + flags.MarkHidden("version") +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go new file mode 100644 index 0000000000..a8980ae1ed --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go @@ -0,0 +1,6 @@ +package config + +type IngressConfig struct { + ServingCertificate []byte + ServingKey []byte +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go new file mode 100644 index 0000000000..1e33b07b96 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go @@ -0,0 +1,24 @@ +package config + +import "path/filepath" + +// KubeConfigID identifies the different kubeconfigs managed in the DataDir +type KubeConfigID string + +const ( + KubeAdmin KubeConfigID = "kubeadmin" + KubeControllerManager KubeConfigID = "kube-controller-manager" + KubeScheduler KubeConfigID = "kube-scheduler" + Kubelet KubeConfigID = "kubelet" + ClusterPolicyController KubeConfigID = "cluster-policy-controller" + RouteControllerManager KubeConfigID = "route-controller-manager" +) + +// KubeConfigPath returns the path to the specified kubeconfig file. +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { + return filepath.Join(dataDir, "resources", string(id), "kubeconfig") +} + +func (cfg *Config) KubeConfigAdminPath(id string) string { + return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go new file mode 100644 index 0000000000..f7e04e5838 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go @@ -0,0 +1,63 @@ +package config + +import ( + "fmt" + "net" + + "github.com/apparentlymart/go-cidr/cidr" +) + +type Network struct { + // IP address pool to use for pod IPs. + // This field is immutable after installation. + ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork"` + + // IP address pool for services. + // Currently, we only support a single entry here. + // This field is immutable after installation. + ServiceNetwork []string `json:"serviceNetwork"` + + // The port range allowed for Services of type NodePort. + // If not specified, the default of 30000-32767 will be used. + // Such Services without a NodePort specified will have one + // automatically allocated from this range. + // This parameter can be updated after the cluster is + // installed. + // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` + ServiceNodePortRange string `json:"serviceNodePortRange"` + + // The DNS server to use + DNS string `json:"-"` +} + +type ClusterNetworkEntry struct { + // The complete block for pod IPs. + CIDR string `json:"cidr"` +} + +func (c *Config) computeAndUpdateClusterDNS() error { + if len(c.Network.ServiceNetwork) == 0 { + return fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to get DNS IP: %v", err) + } + c.Network.DNS = clusterDNS + return nil +} + +// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork +func getClusterDNS(serviceCIDR string) (string, error) { + _, service, err := net.ParseCIDR(serviceCIDR) + if err != nil { + return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) + } + dnsClusterIP, err := cidr.Host(service, 10) + if err != nil { + return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) + } + + return dnsClusterIP.String(), nil +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go new file mode 100644 index 0000000000..6123aa2545 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go @@ -0,0 +1,71 @@ +package config + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strings" + + "k8s.io/klog/v2" +) + +type Node struct { + // If non-empty, will use this string to identify the node instead of the hostname + HostnameOverride string `json:"hostnameOverride"` + + // IP address of the node, passed to the kubelet. + // If not specified, kubelet will use the node's default IP address. + NodeIP string `json:"nodeIP"` +} + +// Determine if the config file specified a NodeName (by default it's assigned the hostname) +func (c *Config) isDefaultNodeName() bool { + hostname, err := os.Hostname() + if err != nil { + klog.Fatalf("Failed to get hostname %v", err) + } + return c.Node.HostnameOverride == strings.ToLower(hostname) +} + +// Read or set the NodeName that will be used for this MicroShift instance +func (c *Config) establishNodeName() (string, error) { + filePath := filepath.Join(GetDataDir(), ".nodename") + contents, err := os.ReadFile(filePath) + if os.IsNotExist(err) { + // ensure that dataDir exists + os.MkdirAll(GetDataDir(), 0700) + if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { + return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) + } + return c.Node.HostnameOverride, nil + } else if err != nil { + return "", err + } + return string(contents), nil +} + +// Validate the NodeName to be used for this MicroShift instances +func (c *Config) validateNodeName(isDefaultNodeName bool) error { + if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) + } + + establishedNodeName, err := c.establishNodeName() + if err != nil { + return fmt.Errorf("failed to establish NodeName: %v", err) + } + + if establishedNodeName != c.Node.HostnameOverride { + if !isDefaultNodeName { + return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", + c.Node.HostnameOverride, establishedNodeName) + } else { + c.Node.HostnameOverride = establishedNodeName + klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ + "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) + } + } + + return nil +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/util.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/util.go new file mode 100644 index 0000000000..6bb69a5079 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/util.go @@ -0,0 +1,21 @@ +package config + +func StringInList(s string, list []string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func stringSliceContains(list []string, elements ...string) bool { + for _, value := range list { + for _, element := range elements { + if value == element { + return true + } + } + } + return false +} diff --git a/pkg/config/apiserver.go b/pkg/config/apiserver.go new file mode 100644 index 0000000000..dcb398c7a0 --- /dev/null +++ b/pkg/config/apiserver.go @@ -0,0 +1,44 @@ +package config + +import ( + "net/url" + "strconv" +) + +type ApiServer struct { + // SubjectAltNames added to API server certs + SubjectAltNames []string `json:"subjectAltNames"` + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. + AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` + + // The URL of the API server + URL string `json:"-"` +} + +// extract the api server port from the cluster URL +func (c *Config) ApiServerPort() (int, error) { + var port string + + parsed, err := url.Parse(c.ApiServer.URL) + if err != nil { + return 0, err + } + + // default empty URL to port 6443 + port = parsed.Port() + if port == "" { + port = "6443" + } + portNum, err := strconv.Atoi(port) + if err != nil { + return 0, err + } + return portNum, nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 7e70bc97e9..376cede347 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,76 +2,22 @@ package config import ( "bytes" - "errors" "fmt" - "net" - "net/url" "os" "os/exec" - "path/filepath" - "strconv" "strings" "time" - "github.com/apparentlymart/go-cidr/cidr" - "github.com/mitchellh/go-homedir" - "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/component-base/logs" "k8s.io/klog/v2" - "sigs.k8s.io/yaml" "github.com/openshift/microshift/pkg/util" ) const ( - DefaultUserConfigFile = "~/.microshift/config.yaml" - defaultUserDataDir = "~/.microshift/data" - DefaultGlobalConfigFile = "/etc/microshift/config.yaml" - defaultGlobalDataDir = "/var/lib/microshift" - // for files managed via management system in /etc, i.e. user applications - defaultManifestDirEtc = "/etc/microshift/manifests" - // for files embedded in ostree. i.e. cni/other component customizations - defaultManifestDirLib = "/usr/lib/microshift/manifests" - // default DNS resolve file when systemd-resolved is used DefaultSystemdResolvedFile = "/run/systemd/resolve/resolv.conf" ) -var ( - configFile = findConfigFile() - dataDir = findDataDir() - manifestsDir = findManifestsDir() -) - -type IngressConfig struct { - ServingCertificate []byte - ServingKey []byte -} - -type EtcdConfig struct { - // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimitMB uint64 `json:"memoryLimitMB"` - - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendBytes int64 `json:"-"` - - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragBytes`, do a defrag. - MinDefragBytes int64 `json:"-"` - MaxFragmentedPercentage float64 `json:"-"` - - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq time.Duration `json:"-"` - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool `json:"-"` -} - -const ( - // Etcd performance degrades significantly if the memory available is less than 50MB, enfore this minimum. - EtcdMinimumMemoryLimit = 50 -) - type Config struct { DNS DNS `json:"dns"` Network Network `json:"network"` @@ -84,112 +30,6 @@ type Config struct { Ingress IngressConfig `json:"-"` } -type Network struct { - // IP address pool to use for pod IPs. - // This field is immutable after installation. - ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork,omitempty"` - - // IP address pool for services. - // Currently, we only support a single entry here. - // This field is immutable after installation. - ServiceNetwork []string `json:"serviceNetwork,omitempty"` - - // The port range allowed for Services of type NodePort. - // If not specified, the default of 30000-32767 will be used. - // Such Services without a NodePort specified will have one - // automatically allocated from this range. - // This parameter can be updated after the cluster is - // installed. - // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` - ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` - - // The DNS server to use - DNS string `json:"-"` -} - -type ClusterNetworkEntry struct { - // The complete block for pod IPs. - CIDR string `json:"cidr,omitempty"` -} - -type DNS struct { - // baseDomain is the base domain of the cluster. All managed DNS records will - // be sub-domains of this base. - // - // For example, given the base domain `example.com`, router exposed - // domains will be formed as `*.apps.example.com` by default, - // and API service will have a DNS entry for `api.example.com`, - // as well as "api-int.example.com" for internal k8s API access. - // - // Once set, this field cannot be changed. - BaseDomain string `json:"baseDomain"` -} - -type ApiServer struct { - // SubjectAltNames added to API server certs - SubjectAltNames []string `json:"subjectAltNames"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - AdvertiseAddress string `json:"advertiseAddress,omitempty"` - // Determines if kube-apiserver controller should configure the - // AdvertiseAddress in the loopback interface. Automatically computed. - SkipInterface bool `json:"-"` - - // The URL of the API server - URL string `json:"-"` -} - -type Node struct { - // If non-empty, will use this string to identify the node instead of the hostname - HostnameOverride string `json:"hostnameOverride"` - - // IP address of the node, passed to the kubelet. - // If not specified, kubelet will use the node's default IP address. - NodeIP string `json:"nodeIP"` -} - -type Debugging struct { - // Valid values are: "Normal", "Debug", "Trace", "TraceAll". - // Defaults to "Normal". - LogLevel string `json:"logLevel"` -} - -func GetConfigFile() string { - return configFile -} - -func GetDataDir() string { - return dataDir -} - -func GetManifestsDir() []string { - return manifestsDir -} - -// KubeConfigID identifies the different kubeconfigs managed in the DataDir -type KubeConfigID string - -const ( - KubeAdmin KubeConfigID = "kubeadmin" - KubeControllerManager KubeConfigID = "kube-controller-manager" - KubeScheduler KubeConfigID = "kube-scheduler" - Kubelet KubeConfigID = "kubelet" - ClusterPolicyController KubeConfigID = "cluster-policy-controller" - RouteControllerManager KubeConfigID = "route-controller-manager" -) - -// KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *Config) KubeConfigPath(id KubeConfigID) string { - return filepath.Join(dataDir, "resources", string(id), "kubeconfig") -} - -func (cfg *Config) KubeConfigAdminPath(id string) string { - return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") -} - var allHostnames []string func getAllHostnames() ([]string, error) { @@ -263,317 +103,3 @@ func NewMicroshiftConfig() *Config { }, } } - -// Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *Config) isDefaultNodeName() bool { - hostname, err := os.Hostname() - if err != nil { - klog.Fatalf("Failed to get hostname %v", err) - } - return c.Node.HostnameOverride == strings.ToLower(hostname) -} - -// Read or set the NodeName that will be used for this MicroShift instance -func (c *Config) establishNodeName() (string, error) { - filePath := filepath.Join(GetDataDir(), ".nodename") - contents, err := os.ReadFile(filePath) - if os.IsNotExist(err) { - // ensure that dataDir exists - os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { - return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) - } - return c.Node.HostnameOverride, nil - } else if err != nil { - return "", err - } - return string(contents), nil -} - -// Validate the NodeName to be used for this MicroShift instances -func (c *Config) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) - } - - establishedNodeName, err := c.establishNodeName() - if err != nil { - return fmt.Errorf("failed to establish NodeName: %v", err) - } - - if establishedNodeName != c.Node.HostnameOverride { - if !isDefaultNodeName { - return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.Node.HostnameOverride, establishedNodeName) - } else { - c.Node.HostnameOverride = establishedNodeName - klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) - } - } - - return nil -} - -// extract the api server port from the cluster URL -func (c *Config) ApiServerPort() (int, error) { - var port string - - parsed, err := url.Parse(c.ApiServer.URL) - if err != nil { - return 0, err - } - - // default empty URL to port 6443 - port = parsed.Port() - if port == "" { - port = "6443" - } - portNum, err := strconv.Atoi(port) - if err != nil { - return 0, err - } - return portNum, nil -} - -// Returns the default user config file if that exists, else the default global -// config file, else the empty string. -func findConfigFile() string { - userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) - if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { - if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { - return "" - } else { - return DefaultGlobalConfigFile - } - } else { - return userConfigFile - } -} - -// Returns the default user data dir if it exists or the user is non-root. -// Returns the default global data dir otherwise. -func findDataDir() string { - userDataDir, _ := homedir.Expand(defaultUserDataDir) - if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { - if os.Geteuid() > 0 { - return userDataDir - } else { - return defaultGlobalDataDir - } - } else { - return userDataDir - } -} - -// Returns the default manifests directories -func findManifestsDir() []string { - var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} - return manifestsDir -} - -func StringInList(s string, list []string) bool { - for _, x := range list { - if x == s { - return true - } - } - return false -} - -func (c *Config) ReadFromConfigFile(configFile string) error { - contents, err := os.ReadFile(configFile) - if err != nil { - return fmt.Errorf("reading config file %q: %v", configFile, err) - } - var config Config - if err := yaml.Unmarshal(contents, &config); err != nil { - return fmt.Errorf("decoding config file %s: %v", configFile, err) - } - - // Wire new Config type to existing Config - c.Node = config.Node - c.Debugging = config.Debugging - c.Network = config.Network - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - c.DNS = config.DNS - c.ApiServer = config.ApiServer - c.ApiServer.URL = "https://localhost:6443" - - if config.Etcd.MemoryLimitMB > 0 { - // If the memory limit is than the minimum, set it to the minimum and continue. - if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { - c.Etcd.MemoryLimitMB = EtcdMinimumMemoryLimit - } else { - c.Etcd.MemoryLimitMB = config.Etcd.MemoryLimitMB - } - } - - return nil -} - -func (c *Config) computeAndUpdateClusterDNS() error { - if len(c.Network.ServiceNetwork) == 0 { - return fmt.Errorf("network.serviceNetwork not filled in") - } - - clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) - if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) - } - c.Network.DNS = clusterDNS - return nil -} - -// Note: add a configFile parameter here because of unit test requiring custom -// local directory -func (c *Config) ReadAndValidate(configFile string) error { - if configFile != "" { - if err := c.ReadFromConfigFile(configFile); err != nil { - return err - } - } - - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - // If KAS advertise address is not configured then compute it from the service - // CIDR automatically. - if len(c.ApiServer.AdvertiseAddress) == 0 { - // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) - // Since the KAS advertise address was not provided we will default to the - // next immediate subnet after the service CIDR. This is due to the fact - // that using the actual apiserver service IP as an endpoint slice breaks - // host network pods trying to reach apiserver, as the VIP 10.43.0.1:443 is - // not translated to 10.43.0.1:6443. It remains unchanged and therefore - // connects to the ingress router instead, triggering all sorts of errors. - nextSubnet, exceed := cidr.NextSubnet(svcNet, 32) - if exceed { - return fmt.Errorf("unable to compute next subnet from service CIDR") - } - // First and last are the same because of the /32 netmask. - firstValidIP, _ := cidr.AddressRange(nextSubnet) - c.ApiServer.AdvertiseAddress = firstValidIP.String() - c.ApiServer.SkipInterface = false - } else { - c.ApiServer.SkipInterface = true - } - - if len(c.ApiServer.SubjectAltNames) > 0 { - // Any entry in SubjectAltNames will be included in the external access certificates. - // Any of the hostnames and IPs (except the node IP) listed below conflicts with - // other certificates, such as the service network and localhost access. - // The node IP is a bit special. Apiserver k8s service, which holds a service IP - // gets resolved to the node IP. If we include the node IP in the SAN then we have - // an ambiguity, the same IP matches two different certificates and there are errors - // when trying to reach apiserver from within the cluster using the service IP. - // Apiserver will decide which certificate to return to client hello based on SNI - // (which client-go does not use) or raw IP mappings. As soon as there is a match for - // the node IP it returns that certificate, which is the external access one. This - // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate - // is invalid. - u, err := url.Parse(c.ApiServer.URL) - if err != nil { - return fmt.Errorf("failed to parse cluster URL: %v", err) - } - if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { - return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") - } - } else { - if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { - return fmt.Errorf("subjectAltNames must not contain node IP") - } - if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { - return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) - } - } - - if stringSliceContains( - c.ApiServer.SubjectAltNames, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc.cluster.local", - "openshift", - "openshift.default", - "openshift.default.svc", - "openshift.default.svc.cluster.local", - c.ApiServer.AdvertiseAddress, - ) { - return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") - } - } - // Validate NodeName in config file, node-name should not be changed for an already - // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage - // being orphaned or lost, and other side effects. - if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { - klog.Fatalf("Error in validating node name: %v", err) - } - - return nil -} - -// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork -func getClusterDNS(serviceCIDR string) (string, error) { - _, service, err := net.ParseCIDR(serviceCIDR) - if err != nil { - return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) - } - dnsClusterIP, err := cidr.Host(service, 10) - if err != nil { - return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) - } - - return dnsClusterIP.String(), nil -} - -func stringSliceContains(list []string, elements ...string) bool { - for _, value := range list { - for _, element := range elements { - if value == element { - return true - } - } - } - return false -} - -// GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { - var verbosity int - switch c.Debugging.LogLevel { - case "Normal": - verbosity = 2 - case "Debug": - verbosity = 4 - case "Trace": - verbosity = 6 - case "TraceAll": - verbosity = 8 - default: - verbosity = 2 - } - return verbosity -} - -func HideUnsupportedFlags(flags *pflag.FlagSet) { - // hide logging flags that we do not use/support - loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) - logs.AddFlags(loggingFlags) - - supportedLoggingFlags := sets.NewString("v") - - loggingFlags.VisitAll(func(pf *pflag.Flag) { - if !supportedLoggingFlags.Has(pf.Name) { - flags.MarkHidden(pf.Name) - } - }) - - flags.MarkHidden("version") -} diff --git a/pkg/config/debugging.go b/pkg/config/debugging.go new file mode 100644 index 0000000000..a540f5490c --- /dev/null +++ b/pkg/config/debugging.go @@ -0,0 +1,25 @@ +package config + +type Debugging struct { + // Valid values are: "Normal", "Debug", "Trace", "TraceAll". + // Defaults to "Normal". + LogLevel string `json:"logLevel"` +} + +// GetVerbosity returns the numerical value for LogLevel which is an enum +func (c *Config) GetVerbosity() int { + var verbosity int + switch c.Debugging.LogLevel { + case "Normal": + verbosity = 2 + case "Debug": + verbosity = 4 + case "Trace": + verbosity = 6 + case "TraceAll": + verbosity = 8 + default: + verbosity = 2 + } + return verbosity +} diff --git a/pkg/config/debugging_test.go b/pkg/config/debugging_test.go new file mode 100644 index 0000000000..2a934370e4 --- /dev/null +++ b/pkg/config/debugging_test.go @@ -0,0 +1,48 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetVerbosity(t *testing.T) { + var ttests = []struct { + setting string + level int + }{ + { + setting: "Normal", + level: 2, + }, + { + setting: "Debug", + level: 4, + }, + { + setting: "Trace", + level: 6, + }, + { + setting: "TraceAll", + level: 8, + }, + { + setting: "Unknown", + level: 2, + }, + { + setting: "", + level: 2, + }, + } + + for _, tt := range ttests { + t.Run(tt.setting, func(t *testing.T) { + config := NewMicroshiftConfig() + config.Debugging.LogLevel = tt.setting + verbosity := config.GetVerbosity() + assert.Equal(t, tt.level, verbosity) + }) + } +} diff --git a/pkg/config/dns.go b/pkg/config/dns.go new file mode 100644 index 0000000000..ca6f11e88e --- /dev/null +++ b/pkg/config/dns.go @@ -0,0 +1,14 @@ +package config + +type DNS struct { + // baseDomain is the base domain of the cluster. All managed DNS records will + // be sub-domains of this base. + // + // For example, given the base domain `example.com`, router exposed + // domains will be formed as `*.apps.example.com` by default, + // and API service will have a DNS entry for `api.example.com`, + // as well as "api-int.example.com" for internal k8s API access. + // + // Once set, this field cannot be changed. + BaseDomain string `json:"baseDomain"` +} diff --git a/pkg/config/etcd.go b/pkg/config/etcd.go new file mode 100644 index 0000000000..dc26b92ec2 --- /dev/null +++ b/pkg/config/etcd.go @@ -0,0 +1,33 @@ +package config + +import "time" + +const ( + // Etcd performance degrades significantly if the memory available + // is less than 50MB, enforce this minimum. + EtcdMinimumMemoryLimit = 50 +) + +type EtcdConfig struct { + // Set a memory limit on the etcd process; etcd will begin paging + // memory when it gets to this value. 0 means no limit. + MemoryLimitMB uint64 `json:"memoryLimitMB"` + + // The limit on the size of the etcd database; etcd will start + // failing writes if its size on disk reaches this value + QuotaBackendBytes int64 `json:"-"` + + // If the backend is fragmented more than + // `maxFragmentedPercentage` and the database size is greater than + // `minDefragBytes`, do a defrag. + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"-"` + + // How often to check the conditions for defragging (0 means no + // defrags, except for a single on startup if `doStartupDefrag` is + // set). + DefragCheckFreq time.Duration `json:"-"` + + // Whether or not to do a defrag when the server finishes starting + DoStartupDefrag bool `json:"-"` +} diff --git a/pkg/config/files.go b/pkg/config/files.go new file mode 100644 index 0000000000..7319033cff --- /dev/null +++ b/pkg/config/files.go @@ -0,0 +1,204 @@ +package config + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + + "github.com/apparentlymart/go-cidr/cidr" + "github.com/mitchellh/go-homedir" + "k8s.io/klog/v2" + "sigs.k8s.io/yaml" +) + +const ( + DefaultUserConfigFile = "~/.microshift/config.yaml" + defaultUserDataDir = "~/.microshift/data" + DefaultGlobalConfigFile = "/etc/microshift/config.yaml" + defaultGlobalDataDir = "/var/lib/microshift" + // for files managed via management system in /etc, i.e. user applications + defaultManifestDirEtc = "/etc/microshift/manifests" + // for files embedded in ostree. i.e. cni/other component customizations + defaultManifestDirLib = "/usr/lib/microshift/manifests" +) + +var ( + configFile = findConfigFile() + dataDir = findDataDir() + manifestsDir = findManifestsDir() +) + +func GetConfigFile() string { + return configFile +} + +func GetDataDir() string { + return dataDir +} + +func GetManifestsDir() []string { + return manifestsDir +} + +// Returns the default user config file if that exists, else the default global +// config file, else the empty string. +func findConfigFile() string { + userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) + if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { + if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { + return "" + } else { + return DefaultGlobalConfigFile + } + } else { + return userConfigFile + } +} + +// Returns the default user data dir if it exists or the user is non-root. +// Returns the default global data dir otherwise. +func findDataDir() string { + userDataDir, _ := homedir.Expand(defaultUserDataDir) + if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { + if os.Geteuid() > 0 { + return userDataDir + } else { + return defaultGlobalDataDir + } + } else { + return userDataDir + } +} + +// Returns the default manifests directories +func findManifestsDir() []string { + var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} + return manifestsDir +} + +func (c *Config) ReadFromConfigFile(configFile string) error { + contents, err := os.ReadFile(configFile) + if err != nil { + return fmt.Errorf("reading config file %q: %v", configFile, err) + } + var config Config + if err := yaml.Unmarshal(contents, &config); err != nil { + return fmt.Errorf("decoding config file %s: %v", configFile, err) + } + + // Wire new Config type to existing Config + c.Node = config.Node + c.Debugging = config.Debugging + c.Network = config.Network + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + } + + c.DNS = config.DNS + c.ApiServer = config.ApiServer + c.ApiServer.URL = "https://localhost:6443" + + if config.Etcd.MemoryLimitMB > 0 { + // If the memory limit is than the minimum, set it to the minimum and continue. + if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { + c.Etcd.MemoryLimitMB = EtcdMinimumMemoryLimit + } else { + c.Etcd.MemoryLimitMB = config.Etcd.MemoryLimitMB + } + } + + return nil +} + +// Note: add a configFile parameter here because of unit test requiring custom +// local directory +func (c *Config) ReadAndValidate(configFile string) error { + if configFile != "" { + if err := c.ReadFromConfigFile(configFile); err != nil { + return err + } + } + + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + } + + // If KAS advertise address is not configured then compute it from the service + // CIDR automatically. + if len(c.ApiServer.AdvertiseAddress) == 0 { + // unchecked error because this was done when getting cluster DNS + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) + // Since the KAS advertise address was not provided we will default to the + // next immediate subnet after the service CIDR. This is due to the fact + // that using the actual apiserver service IP as an endpoint slice breaks + // host network pods trying to reach apiserver, as the VIP 10.43.0.1:443 is + // not translated to 10.43.0.1:6443. It remains unchanged and therefore + // connects to the ingress router instead, triggering all sorts of errors. + nextSubnet, exceed := cidr.NextSubnet(svcNet, 32) + if exceed { + return fmt.Errorf("unable to compute next subnet from service CIDR") + } + // First and last are the same because of the /32 netmask. + firstValidIP, _ := cidr.AddressRange(nextSubnet) + c.ApiServer.AdvertiseAddress = firstValidIP.String() + c.ApiServer.SkipInterface = false + } else { + c.ApiServer.SkipInterface = true + } + + if len(c.ApiServer.SubjectAltNames) > 0 { + // Any entry in SubjectAltNames will be included in the external access certificates. + // Any of the hostnames and IPs (except the node IP) listed below conflicts with + // other certificates, such as the service network and localhost access. + // The node IP is a bit special. Apiserver k8s service, which holds a service IP + // gets resolved to the node IP. If we include the node IP in the SAN then we have + // an ambiguity, the same IP matches two different certificates and there are errors + // when trying to reach apiserver from within the cluster using the service IP. + // Apiserver will decide which certificate to return to client hello based on SNI + // (which client-go does not use) or raw IP mappings. As soon as there is a match for + // the node IP it returns that certificate, which is the external access one. This + // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate + // is invalid. + u, err := url.Parse(c.ApiServer.URL) + if err != nil { + return fmt.Errorf("failed to parse cluster URL: %v", err) + } + if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { + return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") + } + } else { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { + return fmt.Errorf("subjectAltNames must not contain node IP") + } + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) + } + } + + if stringSliceContains( + c.ApiServer.SubjectAltNames, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + "openshift", + "openshift.default", + "openshift.default.svc", + "openshift.default.svc.cluster.local", + c.ApiServer.AdvertiseAddress, + ) { + return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") + } + } + // Validate NodeName in config file, node-name should not be changed for an already + // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage + // being orphaned or lost, and other side effects. + if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { + klog.Fatalf("Error in validating node name: %v", err) + } + + return nil +} diff --git a/pkg/config/flags.go b/pkg/config/flags.go new file mode 100644 index 0000000000..e15ee75a1c --- /dev/null +++ b/pkg/config/flags.go @@ -0,0 +1,23 @@ +package config + +import ( + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/component-base/logs" +) + +func HideUnsupportedFlags(flags *pflag.FlagSet) { + // hide logging flags that we do not use/support + loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) + logs.AddFlags(loggingFlags) + + supportedLoggingFlags := sets.NewString("v") + + loggingFlags.VisitAll(func(pf *pflag.Flag) { + if !supportedLoggingFlags.Has(pf.Name) { + flags.MarkHidden(pf.Name) + } + }) + + flags.MarkHidden("version") +} diff --git a/pkg/config/ingress.go b/pkg/config/ingress.go new file mode 100644 index 0000000000..a8980ae1ed --- /dev/null +++ b/pkg/config/ingress.go @@ -0,0 +1,6 @@ +package config + +type IngressConfig struct { + ServingCertificate []byte + ServingKey []byte +} diff --git a/pkg/config/kubeconfig.go b/pkg/config/kubeconfig.go new file mode 100644 index 0000000000..1e33b07b96 --- /dev/null +++ b/pkg/config/kubeconfig.go @@ -0,0 +1,24 @@ +package config + +import "path/filepath" + +// KubeConfigID identifies the different kubeconfigs managed in the DataDir +type KubeConfigID string + +const ( + KubeAdmin KubeConfigID = "kubeadmin" + KubeControllerManager KubeConfigID = "kube-controller-manager" + KubeScheduler KubeConfigID = "kube-scheduler" + Kubelet KubeConfigID = "kubelet" + ClusterPolicyController KubeConfigID = "cluster-policy-controller" + RouteControllerManager KubeConfigID = "route-controller-manager" +) + +// KubeConfigPath returns the path to the specified kubeconfig file. +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { + return filepath.Join(dataDir, "resources", string(id), "kubeconfig") +} + +func (cfg *Config) KubeConfigAdminPath(id string) string { + return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") +} diff --git a/pkg/config/network.go b/pkg/config/network.go new file mode 100644 index 0000000000..f7e04e5838 --- /dev/null +++ b/pkg/config/network.go @@ -0,0 +1,63 @@ +package config + +import ( + "fmt" + "net" + + "github.com/apparentlymart/go-cidr/cidr" +) + +type Network struct { + // IP address pool to use for pod IPs. + // This field is immutable after installation. + ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork"` + + // IP address pool for services. + // Currently, we only support a single entry here. + // This field is immutable after installation. + ServiceNetwork []string `json:"serviceNetwork"` + + // The port range allowed for Services of type NodePort. + // If not specified, the default of 30000-32767 will be used. + // Such Services without a NodePort specified will have one + // automatically allocated from this range. + // This parameter can be updated after the cluster is + // installed. + // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` + ServiceNodePortRange string `json:"serviceNodePortRange"` + + // The DNS server to use + DNS string `json:"-"` +} + +type ClusterNetworkEntry struct { + // The complete block for pod IPs. + CIDR string `json:"cidr"` +} + +func (c *Config) computeAndUpdateClusterDNS() error { + if len(c.Network.ServiceNetwork) == 0 { + return fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to get DNS IP: %v", err) + } + c.Network.DNS = clusterDNS + return nil +} + +// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork +func getClusterDNS(serviceCIDR string) (string, error) { + _, service, err := net.ParseCIDR(serviceCIDR) + if err != nil { + return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) + } + dnsClusterIP, err := cidr.Host(service, 10) + if err != nil { + return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) + } + + return dnsClusterIP.String(), nil +} diff --git a/pkg/config/node.go b/pkg/config/node.go new file mode 100644 index 0000000000..6123aa2545 --- /dev/null +++ b/pkg/config/node.go @@ -0,0 +1,71 @@ +package config + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strings" + + "k8s.io/klog/v2" +) + +type Node struct { + // If non-empty, will use this string to identify the node instead of the hostname + HostnameOverride string `json:"hostnameOverride"` + + // IP address of the node, passed to the kubelet. + // If not specified, kubelet will use the node's default IP address. + NodeIP string `json:"nodeIP"` +} + +// Determine if the config file specified a NodeName (by default it's assigned the hostname) +func (c *Config) isDefaultNodeName() bool { + hostname, err := os.Hostname() + if err != nil { + klog.Fatalf("Failed to get hostname %v", err) + } + return c.Node.HostnameOverride == strings.ToLower(hostname) +} + +// Read or set the NodeName that will be used for this MicroShift instance +func (c *Config) establishNodeName() (string, error) { + filePath := filepath.Join(GetDataDir(), ".nodename") + contents, err := os.ReadFile(filePath) + if os.IsNotExist(err) { + // ensure that dataDir exists + os.MkdirAll(GetDataDir(), 0700) + if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { + return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) + } + return c.Node.HostnameOverride, nil + } else if err != nil { + return "", err + } + return string(contents), nil +} + +// Validate the NodeName to be used for this MicroShift instances +func (c *Config) validateNodeName(isDefaultNodeName bool) error { + if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) + } + + establishedNodeName, err := c.establishNodeName() + if err != nil { + return fmt.Errorf("failed to establish NodeName: %v", err) + } + + if establishedNodeName != c.Node.HostnameOverride { + if !isDefaultNodeName { + return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", + c.Node.HostnameOverride, establishedNodeName) + } else { + c.Node.HostnameOverride = establishedNodeName + klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ + "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) + } + } + + return nil +} diff --git a/pkg/config/util.go b/pkg/config/util.go new file mode 100644 index 0000000000..6bb69a5079 --- /dev/null +++ b/pkg/config/util.go @@ -0,0 +1,21 @@ +package config + +func StringInList(s string, list []string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func stringSliceContains(list []string, elements ...string) bool { + for _, value := range list { + for _, element := range elements { + if value == element { + return true + } + } + } + return false +} From 0eafbf2a60962464bce705e93db049a413f482d6 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 4 Mar 2023 12:44:26 -0500 Subject: [PATCH 09/17] USHIFT-936: refactor reading and validating of config This change streamlines the reading and validation functions so that it is easier to get a Config instance with the defaults or with valid values read from a file. A new convenience function for loading the active configuration, ignoring a missing configuration file, is added. The k8s YAML parser is used to read the config file, instead of the default parser, to ensure support for `json:"-"` directives to ignore fields in the input. The unit tests are updated to parse input config instead of using a data structure. This more accurately simulates what happens when the real program reads the config if only part of the file is populated, and clarifies which part of the config each test scenario is checking. It also allows for more, smaller, scenarios to isolate configuration validation and defaulting failures. The validation for node name changing is only relevant when starting the server, so that is moved to the run command implementation to make the unit tests for config easier to understand. The show-config command implementation no longer needs to map between multiple configuration data structures, so streamline that code. --- etcd/cmd/microshift-etcd/run.go | 4 +- .../openshift/microshift/pkg/config/config.go | 211 ++++++++--- .../openshift/microshift/pkg/config/files.go | 151 ++------ .../microshift/pkg/config/network.go | 9 +- .../openshift/microshift/pkg/config/node.go | 7 + pkg/cmd/run.go | 16 +- pkg/cmd/showConfig.go | 38 +- pkg/config/config.go | 211 ++++++++--- pkg/config/config_test.go | 339 ++++++++---------- pkg/config/files.go | 151 ++------ pkg/config/network.go | 9 +- pkg/config/node.go | 7 + 12 files changed, 584 insertions(+), 569 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index 2a61c8b408..34f7b1093a 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -22,11 +22,11 @@ import ( ) func NewRunEtcdCommand() *cobra.Command { - cfg := config.NewMicroshiftConfig() cmd := &cobra.Command{ Use: "run", RunE: func(cmd *cobra.Command, args []string) (err error) { - if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { + cfg, err := config.GetActiveConfig() + if err != nil { klog.Fatalf("Error in reading and validating MicroShift config: %v", err) } diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 376cede347..ada8e9694b 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -3,18 +3,21 @@ package config import ( "bytes" "fmt" + "net" + "net/url" "os" "os/exec" "strings" - "time" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" + "github.com/apparentlymart/go-cidr/cidr" "github.com/openshift/microshift/pkg/util" ) const ( + // default DNS resolve file when systemd-resolved is used DefaultSystemdResolvedFile = "/run/systemd/resolve/resolv.conf" ) @@ -30,6 +33,160 @@ type Config struct { Ingress IngressConfig `json:"-"` } +func NewMicroshiftConfig() *Config { + c := &Config{} + err := c.fillDefaults() + if err != nil { + klog.Fatalf("Failed to initialize config: %v", err) + } + return c +} + +func (c *Config) fillDefaults() error { + if c.Debugging.LogLevel == "" { + c.Debugging.LogLevel = "Normal" + } + + if len(c.ApiServer.SubjectAltNames) == 0 { + subjectAltNames, err := getAllHostnames() + if err != nil { + return fmt.Errorf("failed to get all hostnames: %v", err) + } + c.ApiServer.SubjectAltNames = subjectAltNames + } + if c.ApiServer.URL == "" { + c.ApiServer.URL = "https://localhost:6443" + } + + if c.Node.HostnameOverride == "" { + nodeName, err := os.Hostname() + if err != nil { + return fmt.Errorf("Failed to get hostname %v", err) + } + c.Node.HostnameOverride = strings.ToLower(nodeName) + } + if c.Node.NodeIP == "" { + nodeIP, err := util.GetHostIP() + if err != nil { + return fmt.Errorf("failed to get host IP: %v", err) + } + c.Node.NodeIP = nodeIP + } + + if c.DNS.BaseDomain == "" { + c.DNS.BaseDomain = "example.com" + } + + if len(c.Network.ClusterNetwork) == 0 { + c.Network.ClusterNetwork = []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + } + } + if len(c.Network.ServiceNetwork) == 0 { + c.Network.ServiceNetwork = []string{ + "10.43.0.0/16", + } + } + if c.Network.ServiceNodePortRange == "" { + c.Network.ServiceNodePortRange = "30000-32767" + } + if c.Network.DNS == "" { + c.Network.DNS = "10.43.0.10" + } + + return c.updateComputedValues() +} + +func (c *Config) updateComputedValues() error { + clusterDNS, err := c.computeClusterDNS() + if err != nil { + return err + } + c.Network.DNS = clusterDNS + + // If KAS advertise address is not configured then compute it from the service + // CIDR automatically. + if len(c.ApiServer.AdvertiseAddress) == 0 { + // unchecked error because this was done when getting cluster DNS + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) + // Since the KAS advertise address was not provided we will default to the + // next immediate subnet after the service CIDR. This is due to the fact + // that using the actual apiserver service IP as an endpoint slice breaks + // host network pods trying to reach apiserver, as the VIP 10.43.0.1:443 is + // not translated to 10.43.0.1:6443. It remains unchanged and therefore + // connects to the ingress router instead, triggering all sorts of errors. + nextSubnet, exceed := cidr.NextSubnet(svcNet, 32) + if exceed { + return fmt.Errorf("unable to compute next subnet from service CIDR") + } + // First and last are the same because of the /32 netmask. + firstValidIP, _ := cidr.AddressRange(nextSubnet) + c.ApiServer.AdvertiseAddress = firstValidIP.String() + c.ApiServer.SkipInterface = false + } else { + c.ApiServer.SkipInterface = true + } + + return nil +} + +func (c *Config) validate() error { + if len(c.ApiServer.SubjectAltNames) > 0 { + // Any entry in SubjectAltNames will be included in the external access certificates. + // Any of the hostnames and IPs (except the node IP) listed below conflicts with + // other certificates, such as the service network and localhost access. + // The node IP is a bit special. Apiserver k8s service, which holds a service IP + // gets resolved to the node IP. If we include the node IP in the SAN then we have + // an ambiguity, the same IP matches two different certificates and there are errors + // when trying to reach apiserver from within the cluster using the service IP. + // Apiserver will decide which certificate to return to client hello based on SNI + // (which client-go does not use) or raw IP mappings. As soon as there is a match for + // the node IP it returns that certificate, which is the external access one. This + // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate + // is invalid. + u, err := url.Parse(c.ApiServer.URL) + if err != nil { + return fmt.Errorf("failed to parse cluster URL: %v", err) + } + if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { + return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") + } + } else { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { + return fmt.Errorf("subjectAltNames must not contain node IP") + } + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) + } + } + if stringSliceContains( + c.ApiServer.SubjectAltNames, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + "openshift", + "openshift.default", + "openshift.default.svc", + "openshift.default.svc.cluster.local", + c.ApiServer.AdvertiseAddress, + ) { + return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") + } + } + + if c.Etcd.MemoryLimitMB > 0 && c.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { + return fmt.Errorf("etcd.memoryLimitMB value %d is below the minimum allowed %d", + c.Etcd.MemoryLimitMB, EtcdMinimumMemoryLimit, + ) + } + + return nil +} + var allHostnames []string func getAllHostnames() ([]string, error) { @@ -51,55 +208,3 @@ func getAllHostnames() ([]string, error) { allHostnames = set.List() return allHostnames, nil } - -func NewMicroshiftConfig() *Config { - nodeName, err := os.Hostname() - if err != nil { - klog.Fatalf("Failed to get hostname %v", err) - } - nodeIP, err := util.GetHostIP() - if err != nil { - klog.Fatalf("failed to get host IP: %v", err) - } - subjectAltNames, err := getAllHostnames() - if err != nil { - klog.Fatalf("failed to get all hostnames: %v", err) - } - - return &Config{ - Debugging: Debugging{ - LogLevel: "Normal", - }, - ApiServer: ApiServer{ - SubjectAltNames: subjectAltNames, - URL: "https://localhost:6443", - }, - Node: Node{ - HostnameOverride: strings.ToLower(nodeName), - NodeIP: nodeIP, - }, - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.42.0.0/16", - }, - }, - ServiceNetwork: []string{ - "10.43.0.0/16", - }, - ServiceNodePortRange: "30000-32767", - DNS: "10.43.0.10", - }, - Etcd: EtcdConfig{ - MemoryLimitMB: 0, // No limit - MinDefragBytes: 100 * 1024 * 1024, // 100MB - MaxFragmentedPercentage: 45, // percent - DefragCheckFreq: 5 * time.Minute, - DoStartupDefrag: true, - QuotaBackendBytes: 8 * 1024 * 1024 * 1024, // 8GB - }, - } -} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go index 7319033cff..afeca6fa12 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go @@ -3,13 +3,9 @@ package config import ( "errors" "fmt" - "net" - "net/url" "os" - "github.com/apparentlymart/go-cidr/cidr" "github.com/mitchellh/go-homedir" - "k8s.io/klog/v2" "sigs.k8s.io/yaml" ) @@ -78,127 +74,48 @@ func findManifestsDir() []string { return manifestsDir } -func (c *Config) ReadFromConfigFile(configFile string) error { - contents, err := os.ReadFile(configFile) - if err != nil { - return fmt.Errorf("reading config file %q: %v", configFile, err) +func parse(contents []byte) (*Config, error) { + c := &Config{} + fmt.Printf("parsing %s\n", string(contents)) + if err := yaml.Unmarshal(contents, c); err != nil { + return nil, fmt.Errorf("Unable to decode configuration: %v", err) } - var config Config - if err := yaml.Unmarshal(contents, &config); err != nil { - return fmt.Errorf("decoding config file %s: %v", configFile, err) + if err := c.fillDefaults(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - - // Wire new Config type to existing Config - c.Node = config.Node - c.Debugging = config.Debugging - c.Network = config.Network - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + if err := c.updateComputedValues(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - - c.DNS = config.DNS - c.ApiServer = config.ApiServer - c.ApiServer.URL = "https://localhost:6443" - - if config.Etcd.MemoryLimitMB > 0 { - // If the memory limit is than the minimum, set it to the minimum and continue. - if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { - c.Etcd.MemoryLimitMB = EtcdMinimumMemoryLimit - } else { - c.Etcd.MemoryLimitMB = config.Etcd.MemoryLimitMB - } + if err := c.validate(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - - return nil + return c, nil } -// Note: add a configFile parameter here because of unit test requiring custom -// local directory -func (c *Config) ReadAndValidate(configFile string) error { - if configFile != "" { - if err := c.ReadFromConfigFile(configFile); err != nil { - return err - } - } - - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - // If KAS advertise address is not configured then compute it from the service - // CIDR automatically. - if len(c.ApiServer.AdvertiseAddress) == 0 { - // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) - // Since the KAS advertise address was not provided we will default to the - // next immediate subnet after the service CIDR. This is due to the fact - // that using the actual apiserver service IP as an endpoint slice breaks - // host network pods trying to reach apiserver, as the VIP 10.43.0.1:443 is - // not translated to 10.43.0.1:6443. It remains unchanged and therefore - // connects to the ingress router instead, triggering all sorts of errors. - nextSubnet, exceed := cidr.NextSubnet(svcNet, 32) - if exceed { - return fmt.Errorf("unable to compute next subnet from service CIDR") - } - // First and last are the same because of the /32 netmask. - firstValidIP, _ := cidr.AddressRange(nextSubnet) - c.ApiServer.AdvertiseAddress = firstValidIP.String() - c.ApiServer.SkipInterface = false - } else { - c.ApiServer.SkipInterface = true +func Read(configFile string) (*Config, error) { + contents, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("reading config file %q: %v", configFile, err) } + return parse(contents) +} - if len(c.ApiServer.SubjectAltNames) > 0 { - // Any entry in SubjectAltNames will be included in the external access certificates. - // Any of the hostnames and IPs (except the node IP) listed below conflicts with - // other certificates, such as the service network and localhost access. - // The node IP is a bit special. Apiserver k8s service, which holds a service IP - // gets resolved to the node IP. If we include the node IP in the SAN then we have - // an ambiguity, the same IP matches two different certificates and there are errors - // when trying to reach apiserver from within the cluster using the service IP. - // Apiserver will decide which certificate to return to client hello based on SNI - // (which client-go does not use) or raw IP mappings. As soon as there is a match for - // the node IP it returns that certificate, which is the external access one. This - // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate - // is invalid. - u, err := url.Parse(c.ApiServer.URL) - if err != nil { - return fmt.Errorf("failed to parse cluster URL: %v", err) - } - if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { - return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") - } - } else { - if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { - return fmt.Errorf("subjectAltNames must not contain node IP") - } - if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { - return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) - } - } - - if stringSliceContains( - c.ApiServer.SubjectAltNames, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc.cluster.local", - "openshift", - "openshift.default", - "openshift.default.svc", - "openshift.default.svc.cluster.local", - c.ApiServer.AdvertiseAddress, - ) { - return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") - } +// Get the active configuration. If the configuration file exists, +// read it and require it to be valid. Otherwise return the default +// settings. +func GetActiveConfig() (*Config, error) { + var cfg *Config + filename := GetConfigFile() + _, err := os.Stat(filename) + if os.IsNotExist(err) { + // No configuration file, use the default settings + return NewMicroshiftConfig(), nil + } else if err != nil { + return nil, err } - // Validate NodeName in config file, node-name should not be changed for an already - // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage - // being orphaned or lost, and other side effects. - if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { - klog.Fatalf("Error in validating node name: %v", err) + cfg, err = Read(filename) + if err != nil { + return nil, err } - - return nil + return cfg, nil } diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go index f7e04e5838..3a3f8cc211 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go @@ -35,17 +35,16 @@ type ClusterNetworkEntry struct { CIDR string `json:"cidr"` } -func (c *Config) computeAndUpdateClusterDNS() error { +func (c *Config) computeClusterDNS() (string, error) { if len(c.Network.ServiceNetwork) == 0 { - return fmt.Errorf("network.serviceNetwork not filled in") + return "", fmt.Errorf("network.serviceNetwork not filled in") } clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + return "", fmt.Errorf("failed to get DNS IP: %v", err) } - c.Network.DNS = clusterDNS - return nil + return clusterDNS, nil } // getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go index 6123aa2545..cdd6a745dc 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go @@ -69,3 +69,10 @@ func (c *Config) validateNodeName(isDefaultNodeName bool) error { return nil } + +func (c *Config) EnsureNodeNameHasNotChanged() error { + // Validate NodeName in config file, node-name should not be changed for an already + // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage + // being orphaned or lost, and other side effects. + return c.validateNodeName(c.isDefaultNodeName()) +} diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index fe71074c7c..e5880365eb 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -28,12 +28,20 @@ const ( ) func NewRunMicroshiftCommand() *cobra.Command { - cfg := config.NewMicroshiftConfig() - cmd := &cobra.Command{ Use: "run", Short: "Run MicroShift", RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.GetActiveConfig() + if err != nil { + return err + } + // Things to very badly if the node's name has changed + // since the last time the server started. + err = cfg.EnsureNodeNameHasNotChanged() + if err != nil { + return err + } return RunMicroshift(cfg) }, } @@ -42,10 +50,6 @@ func NewRunMicroshiftCommand() *cobra.Command { } func RunMicroshift(cfg *config.Config) error { - if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { - klog.Fatalf("Error in reading or validating configuration: %v", err) - } - // fail early if we don't have enough privileges if os.Geteuid() > 0 { klog.Fatalf("MicroShift must be run privileged") diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 1ec4bbbd96..2a90932b19 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -3,12 +3,11 @@ package cmd import ( "fmt" + "github.com/openshift/microshift/pkg/config" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" cmdutil "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/yaml" - - "github.com/openshift/microshift/pkg/config" ) type showConfigOptions struct { @@ -21,42 +20,23 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command Mode: "default", } - cfg := config.NewMicroshiftConfig() - cmd := &cobra.Command{ Use: "show-config", Short: "Print MicroShift's configuration", Run: func(cmd *cobra.Command, args []string) { + var cfg *config.Config + var err error - switch opts.Mode { - case "default": - cfg.Node.NodeIP = "" - cfg.Node.HostnameOverride = "" - case "effective": - // Load the current configuration - if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { + if opts.Mode == "effective" { + cfg, err = config.GetActiveConfig() + if err != nil { cmdutil.CheckErr(err) } - default: - cmdutil.CheckErr(fmt.Errorf("Unknown mode %q", opts.Mode)) + } else { + cfg = config.NewMicroshiftConfig() } - // map back from internal representation to user config - userCfg := config.Config{ - Network: config.Network{ - ClusterNetwork: []config.ClusterNetworkEntry{ - {CIDR: cfg.Network.ClusterNetwork[0].CIDR}, - }, - ServiceNetwork: []string{cfg.Network.ServiceNetwork[0]}, - ServiceNodePortRange: cfg.Network.ServiceNodePortRange, - }, - DNS: cfg.DNS, - Node: cfg.Node, - ApiServer: cfg.ApiServer, - Debugging: cfg.Debugging, - Etcd: cfg.Etcd, - } - marshalled, err := yaml.Marshal(userCfg) + marshalled, err := yaml.Marshal(cfg) cmdutil.CheckErr(err) fmt.Fprintf(ioStreams.Out, "%s\n", string(marshalled)) diff --git a/pkg/config/config.go b/pkg/config/config.go index 376cede347..ada8e9694b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -3,18 +3,21 @@ package config import ( "bytes" "fmt" + "net" + "net/url" "os" "os/exec" "strings" - "time" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" + "github.com/apparentlymart/go-cidr/cidr" "github.com/openshift/microshift/pkg/util" ) const ( + // default DNS resolve file when systemd-resolved is used DefaultSystemdResolvedFile = "/run/systemd/resolve/resolv.conf" ) @@ -30,6 +33,160 @@ type Config struct { Ingress IngressConfig `json:"-"` } +func NewMicroshiftConfig() *Config { + c := &Config{} + err := c.fillDefaults() + if err != nil { + klog.Fatalf("Failed to initialize config: %v", err) + } + return c +} + +func (c *Config) fillDefaults() error { + if c.Debugging.LogLevel == "" { + c.Debugging.LogLevel = "Normal" + } + + if len(c.ApiServer.SubjectAltNames) == 0 { + subjectAltNames, err := getAllHostnames() + if err != nil { + return fmt.Errorf("failed to get all hostnames: %v", err) + } + c.ApiServer.SubjectAltNames = subjectAltNames + } + if c.ApiServer.URL == "" { + c.ApiServer.URL = "https://localhost:6443" + } + + if c.Node.HostnameOverride == "" { + nodeName, err := os.Hostname() + if err != nil { + return fmt.Errorf("Failed to get hostname %v", err) + } + c.Node.HostnameOverride = strings.ToLower(nodeName) + } + if c.Node.NodeIP == "" { + nodeIP, err := util.GetHostIP() + if err != nil { + return fmt.Errorf("failed to get host IP: %v", err) + } + c.Node.NodeIP = nodeIP + } + + if c.DNS.BaseDomain == "" { + c.DNS.BaseDomain = "example.com" + } + + if len(c.Network.ClusterNetwork) == 0 { + c.Network.ClusterNetwork = []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + } + } + if len(c.Network.ServiceNetwork) == 0 { + c.Network.ServiceNetwork = []string{ + "10.43.0.0/16", + } + } + if c.Network.ServiceNodePortRange == "" { + c.Network.ServiceNodePortRange = "30000-32767" + } + if c.Network.DNS == "" { + c.Network.DNS = "10.43.0.10" + } + + return c.updateComputedValues() +} + +func (c *Config) updateComputedValues() error { + clusterDNS, err := c.computeClusterDNS() + if err != nil { + return err + } + c.Network.DNS = clusterDNS + + // If KAS advertise address is not configured then compute it from the service + // CIDR automatically. + if len(c.ApiServer.AdvertiseAddress) == 0 { + // unchecked error because this was done when getting cluster DNS + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) + // Since the KAS advertise address was not provided we will default to the + // next immediate subnet after the service CIDR. This is due to the fact + // that using the actual apiserver service IP as an endpoint slice breaks + // host network pods trying to reach apiserver, as the VIP 10.43.0.1:443 is + // not translated to 10.43.0.1:6443. It remains unchanged and therefore + // connects to the ingress router instead, triggering all sorts of errors. + nextSubnet, exceed := cidr.NextSubnet(svcNet, 32) + if exceed { + return fmt.Errorf("unable to compute next subnet from service CIDR") + } + // First and last are the same because of the /32 netmask. + firstValidIP, _ := cidr.AddressRange(nextSubnet) + c.ApiServer.AdvertiseAddress = firstValidIP.String() + c.ApiServer.SkipInterface = false + } else { + c.ApiServer.SkipInterface = true + } + + return nil +} + +func (c *Config) validate() error { + if len(c.ApiServer.SubjectAltNames) > 0 { + // Any entry in SubjectAltNames will be included in the external access certificates. + // Any of the hostnames and IPs (except the node IP) listed below conflicts with + // other certificates, such as the service network and localhost access. + // The node IP is a bit special. Apiserver k8s service, which holds a service IP + // gets resolved to the node IP. If we include the node IP in the SAN then we have + // an ambiguity, the same IP matches two different certificates and there are errors + // when trying to reach apiserver from within the cluster using the service IP. + // Apiserver will decide which certificate to return to client hello based on SNI + // (which client-go does not use) or raw IP mappings. As soon as there is a match for + // the node IP it returns that certificate, which is the external access one. This + // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate + // is invalid. + u, err := url.Parse(c.ApiServer.URL) + if err != nil { + return fmt.Errorf("failed to parse cluster URL: %v", err) + } + if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { + return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") + } + } else { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { + return fmt.Errorf("subjectAltNames must not contain node IP") + } + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) + } + } + if stringSliceContains( + c.ApiServer.SubjectAltNames, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + "openshift", + "openshift.default", + "openshift.default.svc", + "openshift.default.svc.cluster.local", + c.ApiServer.AdvertiseAddress, + ) { + return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") + } + } + + if c.Etcd.MemoryLimitMB > 0 && c.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { + return fmt.Errorf("etcd.memoryLimitMB value %d is below the minimum allowed %d", + c.Etcd.MemoryLimitMB, EtcdMinimumMemoryLimit, + ) + } + + return nil +} + var allHostnames []string func getAllHostnames() ([]string, error) { @@ -51,55 +208,3 @@ func getAllHostnames() ([]string, error) { allHostnames = set.List() return allHostnames, nil } - -func NewMicroshiftConfig() *Config { - nodeName, err := os.Hostname() - if err != nil { - klog.Fatalf("Failed to get hostname %v", err) - } - nodeIP, err := util.GetHostIP() - if err != nil { - klog.Fatalf("failed to get host IP: %v", err) - } - subjectAltNames, err := getAllHostnames() - if err != nil { - klog.Fatalf("failed to get all hostnames: %v", err) - } - - return &Config{ - Debugging: Debugging{ - LogLevel: "Normal", - }, - ApiServer: ApiServer{ - SubjectAltNames: subjectAltNames, - URL: "https://localhost:6443", - }, - Node: Node{ - HostnameOverride: strings.ToLower(nodeName), - NodeIP: nodeIP, - }, - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.42.0.0/16", - }, - }, - ServiceNetwork: []string{ - "10.43.0.0/16", - }, - ServiceNodePortRange: "30000-32767", - DNS: "10.43.0.10", - }, - Etcd: EtcdConfig{ - MemoryLimitMB: 0, // No limit - MinDefragBytes: 100 * 1024 * 1024, // 100MB - MaxFragmentedPercentage: 45, // percent - DefragCheckFreq: 5 * time.Minute, - DoStartupDefrag: true, - QuotaBackendBytes: 8 * 1024 * 1024 * 1024, // 8GB - }, - } -} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 6b98e908be..1e8d8696d9 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -4,9 +4,6 @@ import ( "os" "path/filepath" "testing" - "time" - - "sigs.k8s.io/yaml" "github.com/stretchr/testify/assert" ) @@ -27,93 +24,119 @@ func setupSuiteDataDir(t *testing.T) func() { } } -func TestConfigFile(t *testing.T) { +func TestParse(t *testing.T) { + mkConfig := func() *Config { + c := NewMicroshiftConfig() + c.ApiServer.SkipInterface = true + return c + } + var ttests = []struct { - config Config - expected Config + name string + config string + expected *Config expectErr bool }{ { - config: Config{ - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.20.30.40/16", - }, - }, - ServiceNetwork: []string{"40.30.20.10/16"}, - ServiceNodePortRange: "1024-32767", - }, - Node: Node{ - HostnameOverride: "node1", - NodeIP: "1.2.3.4", - }, - ApiServer: ApiServer{ - SubjectAltNames: []string{"node1", "node2"}, - AdvertiseAddress: "6.7.8.9", - }, - Debugging: Debugging{ - LogLevel: "Debug", - }, - }, - expected: Config{ - Debugging: Debugging{ - LogLevel: "Debug", - }, - ApiServer: ApiServer{ - SubjectAltNames: []string{"node1", "node2"}, - AdvertiseAddress: "6.7.8.9", - URL: "https://localhost:6443", - }, - Node: Node{ - HostnameOverride: "node1", - NodeIP: "1.2.3.4", - }, - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.20.30.40/16", - }, + name: "empty", + config: "", + expected: mkConfig(), + }, + { + name: "dns", + config: ` +dns: + baseDomain: test-example.com +`, + expected: func() *Config { + c := mkConfig() + c.DNS.BaseDomain = "test-example.com" + return c + }(), + }, + { + name: "network", + config: ` +network: + clusterNetwork: + - cidr: "10.20.30.40/16" + serviceNetwork: + - "40.30.20.10/16" + serviceNodePortRange: "1024-32767" +`, + expected: func() *Config { + c := mkConfig() + c.Network.ClusterNetwork = []ClusterNetworkEntry{ + { + CIDR: "10.20.30.40/16", }, - ServiceNetwork: []string{"40.30.20.10/16"}, - ServiceNodePortRange: "1024-32767", - DNS: "40.30.0.10", - }, - Etcd: EtcdConfig{ - MemoryLimitMB: 0, - QuotaBackendBytes: 8 * 1024 * 1024 * 1024, - MinDefragBytes: 100 * 1024 * 1024, - MaxFragmentedPercentage: 45, - DefragCheckFreq: 5 * time.Minute, - DoStartupDefrag: true, - }, - }, - expectErr: false, + } + c.Network.ServiceNetwork = []string{"40.30.20.10/16"} + c.Network.ServiceNodePortRange = "1024-32767" + c.ApiServer.AdvertiseAddress = "40.31.0.0" // computed default + c.updateComputedValues() // recomputes DNS field + return c + }(), + }, + { + name: "node", + config: ` +node: + hostnameOverride: "node1" + nodeIP: "1.2.3.4" +`, + expected: func() *Config { + c := mkConfig() + c.Node.HostnameOverride = "node1" + c.Node.NodeIP = "1.2.3.4" + return c + }(), + }, + { + name: "api-server", + config: ` +apiServer: + subjectAltNames: + - node1 + - node2 +`, + expected: func() *Config { + c := mkConfig() + c.ApiServer.SubjectAltNames = []string{ + "node1", "node2", + } + return c + }(), + }, + { + name: "debugging", + config: ` +debugging: + logLevel: Info +`, + expected: func() *Config { + c := mkConfig() + c.Debugging.LogLevel = "Info" + return c + }(), + }, + { + name: "etcd", + config: ` +etcd: + memoryLimitMB: 100 +`, + expected: func() *Config { + c := mkConfig() + c.Etcd.MemoryLimitMB = 100 + return c + }(), }, } + for _, tt := range ttests { - t.Run("", func(t *testing.T) { - f, err := os.CreateTemp("", "test") - if err != nil { - t.Errorf("unable to create temp file: %v", err) - } - defer os.Remove(f.Name()) - d, err := yaml.Marshal(&tt.config) - if err != nil { - t.Errorf("unable to marshal configuration: %v", err) - } - _, err = f.Write(d) - if err != nil { - t.Errorf("unable to write to file: %v", err) - } - config := NewMicroshiftConfig() - err = config.ReadFromConfigFile(f.Name()) + t.Run(tt.name, func(t *testing.T) { + config, err := parse([]byte(tt.config)) if tt.expectErr && err == nil { t.Fatal("Expecting error and received nothing") } @@ -121,136 +144,88 @@ func TestConfigFile(t *testing.T) { t.Fatalf("Not expecting error and received: %v", err) } if !tt.expectErr { - assert.Equal(t, tt.expected, *config) + assert.Equal(t, tt.expected, config) } }) } } -// test the MicroshiftConfig.ReadAndValidate function to verify that it configures MicroshiftConfig from -// a configuration file. -func TestMicroshiftConfigReadAndValidate(t *testing.T) { +// Test the validation logic +func TestValidate(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() + mkConfig := func() *Config { + c := NewMicroshiftConfig() + c.ApiServer.SkipInterface = true + return c + } + var ttests = []struct { name string - config Config - expected Config + config *Config expectErr bool }{ { - name: "Config OK full", - config: Config{ - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.20.30.40/16", - }, - }, - ServiceNetwork: []string{"40.30.20.10/16"}, - ServiceNodePortRange: "1024-32767", - }, - Node: Node{ - HostnameOverride: "node1", - NodeIP: "1.2.3.4", - }, - ApiServer: ApiServer{ - SubjectAltNames: []string{"node1", "node2"}, - AdvertiseAddress: "6.7.8.9", - }, - Debugging: Debugging{ - LogLevel: "Debug", - }, - }, - expected: Config{ - Debugging: Debugging{ - LogLevel: "Debug", - }, - ApiServer: ApiServer{ - SubjectAltNames: []string{"node1", "node2"}, - AdvertiseAddress: "6.7.8.9", - SkipInterface: true, - URL: "https://localhost:6443", - }, - Node: Node{ - HostnameOverride: "node1", - NodeIP: "1.2.3.4", - }, - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.20.30.40/16", - }, - }, - ServiceNetwork: []string{"40.30.20.10/16"}, - ServiceNodePortRange: "1024-32767", - DNS: "40.30.0.10", - }, - Etcd: EtcdConfig{ - MemoryLimitMB: 0, - QuotaBackendBytes: 8 * 1024 * 1024 * 1024, - MinDefragBytes: 100 * 1024 * 1024, - MaxFragmentedPercentage: 45, - DefragCheckFreq: 5 * time.Minute, - DoStartupDefrag: true, - }, - }, + name: "defaults-ok", + config: NewMicroshiftConfig(), expectErr: false, }, { - name: "Config NOK with bad SAN localhost", - config: Config{ - ApiServer: ApiServer{ - SubjectAltNames: []string{"127.0.0.1", "localhost"}, - }, - }, - expected: Config{}, + name: "subject-alt-names-with-localhost", + config: func() *Config { + c := mkConfig() + c.ApiServer.SubjectAltNames = []string{"localhost"} + return c + }(), + expectErr: true, + }, + { + name: "subject-alt-names-with-loopback-ipv4", + config: func() *Config { + c := mkConfig() + c.ApiServer.SubjectAltNames = []string{"127.0.0.1"} + return c + }(), + expectErr: true, + }, + { + name: "subject-alt-names-with-kubernetes", + config: func() *Config { + c := mkConfig() + c.ApiServer.SubjectAltNames = []string{"kubernetes"} + return c + }(), expectErr: true, }, { - name: "Config NOK with bad SAN kubernetes service", - config: Config{ - ApiServer: ApiServer{ - SubjectAltNames: []string{"kubernetes"}, - }, - }, - expected: Config{}, + name: "etcd-memory-limit-low", + config: func() *Config { + c := mkConfig() + c.Etcd.MemoryLimitMB = 1 + return c + }(), expectErr: true, }, + { + name: "etcd-memory-zero", + config: func() *Config { + c := mkConfig() + c.Etcd.MemoryLimitMB = 0 + return c + }(), + expectErr: false, + }, } for _, tt := range ttests { t.Run(tt.name, func(t *testing.T) { - f, err := os.CreateTemp("", "test") - if err != nil { - t.Errorf("unable to create temp file: %v", err) - } - defer os.Remove(f.Name()) - d, err := yaml.Marshal(&tt.config) - if err != nil { - t.Errorf("unable to marshal configuration: %v", err) - } - _, err = f.Write(d) - if err != nil { - t.Errorf("unable to write to file: %v", err) - } - config := NewMicroshiftConfig() - err = config.ReadAndValidate(f.Name()) + err := tt.config.validate() if tt.expectErr && err == nil { t.Fatal("Expecting error and received nothing") } if !tt.expectErr && err != nil { t.Fatalf("Not expecting error and received: %v", err) } - if !tt.expectErr { - assert.Equal(t, tt.expected, *config) - } }) } } diff --git a/pkg/config/files.go b/pkg/config/files.go index 7319033cff..afeca6fa12 100644 --- a/pkg/config/files.go +++ b/pkg/config/files.go @@ -3,13 +3,9 @@ package config import ( "errors" "fmt" - "net" - "net/url" "os" - "github.com/apparentlymart/go-cidr/cidr" "github.com/mitchellh/go-homedir" - "k8s.io/klog/v2" "sigs.k8s.io/yaml" ) @@ -78,127 +74,48 @@ func findManifestsDir() []string { return manifestsDir } -func (c *Config) ReadFromConfigFile(configFile string) error { - contents, err := os.ReadFile(configFile) - if err != nil { - return fmt.Errorf("reading config file %q: %v", configFile, err) +func parse(contents []byte) (*Config, error) { + c := &Config{} + fmt.Printf("parsing %s\n", string(contents)) + if err := yaml.Unmarshal(contents, c); err != nil { + return nil, fmt.Errorf("Unable to decode configuration: %v", err) } - var config Config - if err := yaml.Unmarshal(contents, &config); err != nil { - return fmt.Errorf("decoding config file %s: %v", configFile, err) + if err := c.fillDefaults(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - - // Wire new Config type to existing Config - c.Node = config.Node - c.Debugging = config.Debugging - c.Network = config.Network - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + if err := c.updateComputedValues(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - - c.DNS = config.DNS - c.ApiServer = config.ApiServer - c.ApiServer.URL = "https://localhost:6443" - - if config.Etcd.MemoryLimitMB > 0 { - // If the memory limit is than the minimum, set it to the minimum and continue. - if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { - c.Etcd.MemoryLimitMB = EtcdMinimumMemoryLimit - } else { - c.Etcd.MemoryLimitMB = config.Etcd.MemoryLimitMB - } + if err := c.validate(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - - return nil + return c, nil } -// Note: add a configFile parameter here because of unit test requiring custom -// local directory -func (c *Config) ReadAndValidate(configFile string) error { - if configFile != "" { - if err := c.ReadFromConfigFile(configFile); err != nil { - return err - } - } - - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - // If KAS advertise address is not configured then compute it from the service - // CIDR automatically. - if len(c.ApiServer.AdvertiseAddress) == 0 { - // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) - // Since the KAS advertise address was not provided we will default to the - // next immediate subnet after the service CIDR. This is due to the fact - // that using the actual apiserver service IP as an endpoint slice breaks - // host network pods trying to reach apiserver, as the VIP 10.43.0.1:443 is - // not translated to 10.43.0.1:6443. It remains unchanged and therefore - // connects to the ingress router instead, triggering all sorts of errors. - nextSubnet, exceed := cidr.NextSubnet(svcNet, 32) - if exceed { - return fmt.Errorf("unable to compute next subnet from service CIDR") - } - // First and last are the same because of the /32 netmask. - firstValidIP, _ := cidr.AddressRange(nextSubnet) - c.ApiServer.AdvertiseAddress = firstValidIP.String() - c.ApiServer.SkipInterface = false - } else { - c.ApiServer.SkipInterface = true +func Read(configFile string) (*Config, error) { + contents, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("reading config file %q: %v", configFile, err) } + return parse(contents) +} - if len(c.ApiServer.SubjectAltNames) > 0 { - // Any entry in SubjectAltNames will be included in the external access certificates. - // Any of the hostnames and IPs (except the node IP) listed below conflicts with - // other certificates, such as the service network and localhost access. - // The node IP is a bit special. Apiserver k8s service, which holds a service IP - // gets resolved to the node IP. If we include the node IP in the SAN then we have - // an ambiguity, the same IP matches two different certificates and there are errors - // when trying to reach apiserver from within the cluster using the service IP. - // Apiserver will decide which certificate to return to client hello based on SNI - // (which client-go does not use) or raw IP mappings. As soon as there is a match for - // the node IP it returns that certificate, which is the external access one. This - // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate - // is invalid. - u, err := url.Parse(c.ApiServer.URL) - if err != nil { - return fmt.Errorf("failed to parse cluster URL: %v", err) - } - if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { - return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") - } - } else { - if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { - return fmt.Errorf("subjectAltNames must not contain node IP") - } - if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { - return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) - } - } - - if stringSliceContains( - c.ApiServer.SubjectAltNames, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc.cluster.local", - "openshift", - "openshift.default", - "openshift.default.svc", - "openshift.default.svc.cluster.local", - c.ApiServer.AdvertiseAddress, - ) { - return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") - } +// Get the active configuration. If the configuration file exists, +// read it and require it to be valid. Otherwise return the default +// settings. +func GetActiveConfig() (*Config, error) { + var cfg *Config + filename := GetConfigFile() + _, err := os.Stat(filename) + if os.IsNotExist(err) { + // No configuration file, use the default settings + return NewMicroshiftConfig(), nil + } else if err != nil { + return nil, err } - // Validate NodeName in config file, node-name should not be changed for an already - // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage - // being orphaned or lost, and other side effects. - if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { - klog.Fatalf("Error in validating node name: %v", err) + cfg, err = Read(filename) + if err != nil { + return nil, err } - - return nil + return cfg, nil } diff --git a/pkg/config/network.go b/pkg/config/network.go index f7e04e5838..3a3f8cc211 100644 --- a/pkg/config/network.go +++ b/pkg/config/network.go @@ -35,17 +35,16 @@ type ClusterNetworkEntry struct { CIDR string `json:"cidr"` } -func (c *Config) computeAndUpdateClusterDNS() error { +func (c *Config) computeClusterDNS() (string, error) { if len(c.Network.ServiceNetwork) == 0 { - return fmt.Errorf("network.serviceNetwork not filled in") + return "", fmt.Errorf("network.serviceNetwork not filled in") } clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + return "", fmt.Errorf("failed to get DNS IP: %v", err) } - c.Network.DNS = clusterDNS - return nil + return clusterDNS, nil } // getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork diff --git a/pkg/config/node.go b/pkg/config/node.go index 6123aa2545..cdd6a745dc 100644 --- a/pkg/config/node.go +++ b/pkg/config/node.go @@ -69,3 +69,10 @@ func (c *Config) validateNodeName(isDefaultNodeName bool) error { return nil } + +func (c *Config) EnsureNodeNameHasNotChanged() error { + // Validate NodeName in config file, node-name should not be changed for an already + // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage + // being orphaned or lost, and other side effects. + return c.validateNodeName(c.isDefaultNodeName()) +} From 5c14fcb6c1671b8f53e4037fa0be521eb2b5de70 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 6 Mar 2023 18:07:19 -0500 Subject: [PATCH 10/17] USHIFT-936: show configuration on startup --- pkg/cmd/run.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index e5880365eb..1cfc08eaed 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -4,6 +4,7 @@ import ( "context" "os" "os/signal" + "strings" "syscall" "time" @@ -21,6 +22,7 @@ import ( "github.com/spf13/cobra" "k8s.io/klog/v2" + "sigs.k8s.io/yaml" ) const ( @@ -49,12 +51,25 @@ func NewRunMicroshiftCommand() *cobra.Command { return cmd } +func logConfig(cfg *config.Config) { + marshalled, err := yaml.Marshal(cfg) + if err != nil { + klog.Fatal(err) + } + klog.Info("Effective configuration:") + for _, line := range strings.Split(string(marshalled), "\n") { + klog.Info(line) + } +} + func RunMicroshift(cfg *config.Config) error { // fail early if we don't have enough privileges if os.Geteuid() > 0 { klog.Fatalf("MicroShift must be run privileged") } + logConfig(cfg) + // TO-DO: When multi-node is ready, we need to add the controller host-name/mDNS hostname // or VIP to this list on start // see https://github.com/openshift/microshift/pull/471 From 27e446f8d5a05af2b43e225f0211ecd7a1e02ee1 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 8 Mar 2023 10:27:46 -0500 Subject: [PATCH 11/17] USHIFT-936: separate user values from defaults and computed values Track the values we read from the user-provided configuration file separately from those that are defaulted or computed from other settings. This allows us to apply rules to computed values, like ApiServer.SkipAddress, based on whether or not the user has overridden the default. --- .../openshift/microshift/pkg/config/config.go | 165 +++++++++++++----- .../openshift/microshift/pkg/config/files.go | 39 +++-- pkg/config/config.go | 165 +++++++++++++----- pkg/config/config_test.go | 151 ++++++++++------ pkg/config/files.go | 39 +++-- 5 files changed, 381 insertions(+), 178 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index ada8e9694b..7939ee5858 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "strings" + "time" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" @@ -30,84 +31,159 @@ type Config struct { Debugging Debugging `json:"debugging"` // Internal-only fields - Ingress IngressConfig `json:"-"` + Ingress IngressConfig `json:"-"` + userSettings *Config `json:"-"` // the values read from the config file } +// NewMicroshiftConfig creates a new Config struct populated with the +// default values and with any computed values updated based on those +// defaults. func NewMicroshiftConfig() *Config { c := &Config{} - err := c.fillDefaults() - if err != nil { + if err := c.fillDefaults(); err != nil { + klog.Fatalf("Failed to initialize config: %v", err) + } + if err := c.updateComputedValues(); err != nil { klog.Fatalf("Failed to initialize config: %v", err) } return c } +// fillDefaults forcibly sets the configuration to the default +// values. We do not use a static struct for the defaults because some +// of them are computed from the environment. If any error occurs +// probing the environment, the values in the Config instance are not +// changed. func (c *Config) fillDefaults() error { - if c.Debugging.LogLevel == "" { - c.Debugging.LogLevel = "Normal" - } - if len(c.ApiServer.SubjectAltNames) == 0 { - subjectAltNames, err := getAllHostnames() - if err != nil { - return fmt.Errorf("failed to get all hostnames: %v", err) - } - c.ApiServer.SubjectAltNames = subjectAltNames + // Look up any values that may generate an error + subjectAltNames, err := getAllHostnames() + if err != nil { + return fmt.Errorf("failed to get all hostnames: %v", err) } - if c.ApiServer.URL == "" { - c.ApiServer.URL = "https://localhost:6443" + hostname, err := os.Hostname() + if err != nil { + return fmt.Errorf("Failed to get hostname %v", err) + } + nodeIP, err := util.GetHostIP() + if err != nil { + return fmt.Errorf("failed to get host IP: %v", err) } - if c.Node.HostnameOverride == "" { - nodeName, err := os.Hostname() - if err != nil { - return fmt.Errorf("Failed to get hostname %v", err) - } - c.Node.HostnameOverride = strings.ToLower(nodeName) + c.Debugging = Debugging{ + LogLevel: "Normal", } - if c.Node.NodeIP == "" { - nodeIP, err := util.GetHostIP() - if err != nil { - return fmt.Errorf("failed to get host IP: %v", err) - } - c.Node.NodeIP = nodeIP + c.ApiServer = ApiServer{ + SubjectAltNames: subjectAltNames, + URL: "https://localhost:6443", } - - if c.DNS.BaseDomain == "" { - c.DNS.BaseDomain = "example.com" + c.Node = Node{ + HostnameOverride: hostname, + NodeIP: nodeIP, } - - if len(c.Network.ClusterNetwork) == 0 { - c.Network.ClusterNetwork = []ClusterNetworkEntry{ + c.DNS = DNS{ + BaseDomain: "example.com", + } + c.Network = Network{ + ClusterNetwork: []ClusterNetworkEntry{ { CIDR: "10.42.0.0/16", }, - } - } - if len(c.Network.ServiceNetwork) == 0 { - c.Network.ServiceNetwork = []string{ + }, + ServiceNetwork: []string{ "10.43.0.0/16", + }, + ServiceNodePortRange: "30000-32767", + DNS: "10.43.0.10", + } + c.Etcd = EtcdConfig{ + MemoryLimitMB: 0, + DoStartupDefrag: true, // FIXME: flip this to SkipStartupDefrag + QuotaBackendBytes: 8 * 1024 * 1024 * 1024, + MinDefragBytes: 100 * 1024 * 1024, + MaxFragmentedPercentage: 45, + DefragCheckFreq: 5 * time.Minute, + } + + return nil +} + +// incorporateUserSettings merges any values read from the +// configuration file provided by the user with the existing settings +// (usually the defaults). +func (c *Config) incorporateUserSettings(u *Config) { + c.userSettings = u + + if u.DNS.BaseDomain != "" { + c.DNS.BaseDomain = u.DNS.BaseDomain + } + + if len(u.Network.ClusterNetwork) != 0 { + c.Network.ClusterNetwork = u.Network.ClusterNetwork + } + if len(u.Network.ServiceNetwork) != 0 { + c.Network.ServiceNetwork = u.Network.ServiceNetwork + // The default for the API server address is computed from the + // service network. If the user provides a network without + // also overriding the computed address, we need to clear the + // address here so it is recomputed later. If they provide + // both the network and the address, the address will be + // copied into place below with the other API server settings. + if u.ApiServer.AdvertiseAddress == "" { + c.ApiServer.AdvertiseAddress = "" } } - if c.Network.ServiceNodePortRange == "" { - c.Network.ServiceNodePortRange = "30000-32767" + if u.Network.ServiceNodePortRange != "" { + c.Network.ServiceNodePortRange = u.Network.ServiceNodePortRange + } + if u.Network.DNS != "" { + c.Network.DNS = u.Network.DNS + } + + if u.Etcd.MemoryLimitMB != 0 { + c.Etcd.MemoryLimitMB = u.Etcd.MemoryLimitMB + } + + if u.Node.HostnameOverride != "" { + c.Node.HostnameOverride = u.Node.HostnameOverride + } + if u.Node.NodeIP != "" { + c.Node.NodeIP = u.Node.NodeIP + } + + if len(u.ApiServer.SubjectAltNames) != 0 { + c.ApiServer.SubjectAltNames = u.ApiServer.SubjectAltNames + } + if u.ApiServer.AdvertiseAddress != "" { + c.ApiServer.AdvertiseAddress = u.ApiServer.AdvertiseAddress } - if c.Network.DNS == "" { - c.Network.DNS = "10.43.0.10" + if u.ApiServer.URL != "" { + c.ApiServer.URL = u.ApiServer.URL } - return c.updateComputedValues() + if u.Debugging.LogLevel != "" { + c.Debugging.LogLevel = u.Debugging.LogLevel + } } +// updateComputedValues examins the existing settings and converts any +// inputs to more easily consumable units or fills in any defaults +// computed based on the values of other settings. func (c *Config) updateComputedValues() error { + clusterDNS, err := c.computeClusterDNS() if err != nil { return err } c.Network.DNS = clusterDNS - // If KAS advertise address is not configured then compute it from the service - // CIDR automatically. + // If KAS advertise address configured, we do not want to apply + // the IP to the internal interface. + if c.userSettings != nil && len(c.userSettings.ApiServer.AdvertiseAddress) != 0 { + c.ApiServer.SkipInterface = true + } + + // If we have no advertise address, pick one. if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) @@ -124,9 +200,6 @@ func (c *Config) updateComputedValues() error { // First and last are the same because of the /32 netmask. firstValidIP, _ := cidr.AddressRange(nextSubnet) c.ApiServer.AdvertiseAddress = firstValidIP.String() - c.ApiServer.SkipInterface = false - } else { - c.ApiServer.SkipInterface = true } return nil diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go index afeca6fa12..3b8abfd44b 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go @@ -76,35 +76,38 @@ func findManifestsDir() []string { func parse(contents []byte) (*Config, error) { c := &Config{} - fmt.Printf("parsing %s\n", string(contents)) if err := yaml.Unmarshal(contents, c); err != nil { return nil, fmt.Errorf("Unable to decode configuration: %v", err) } - if err := c.fillDefaults(); err != nil { - return nil, fmt.Errorf("Invalid configuration: %v", err) + return c, nil +} + +func getActiveConfigFromYAML(contents []byte) (*Config, error) { + userSettings, err := parse(contents) + if err != nil { + return nil, fmt.Errorf("Error parsing config file %q: %v", configFile, err) } - if err := c.updateComputedValues(); err != nil { + + // Start with the defaults, then apply the user settings and + // recompute dynamic values. + results := &Config{} + if err := results.fillDefaults(); err != nil { return nil, fmt.Errorf("Invalid configuration: %v", err) } - if err := c.validate(); err != nil { + results.incorporateUserSettings(userSettings) + if err := results.updateComputedValues(); err != nil { return nil, fmt.Errorf("Invalid configuration: %v", err) } - return c, nil -} - -func Read(configFile string) (*Config, error) { - contents, err := os.ReadFile(configFile) - if err != nil { - return nil, fmt.Errorf("reading config file %q: %v", configFile, err) + if err := results.validate(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - return parse(contents) + return results, nil } // Get the active configuration. If the configuration file exists, // read it and require it to be valid. Otherwise return the default // settings. func GetActiveConfig() (*Config, error) { - var cfg *Config filename := GetConfigFile() _, err := os.Stat(filename) if os.IsNotExist(err) { @@ -113,9 +116,11 @@ func GetActiveConfig() (*Config, error) { } else if err != nil { return nil, err } - cfg, err = Read(filename) + + // Read the file and merge user-provided settings with the defaults + contents, err := os.ReadFile(configFile) if err != nil { - return nil, err + return nil, fmt.Errorf("Error reading config file %q: %v", configFile, err) } - return cfg, nil + return getActiveConfigFromYAML(contents) } diff --git a/pkg/config/config.go b/pkg/config/config.go index ada8e9694b..7939ee5858 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "strings" + "time" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" @@ -30,84 +31,159 @@ type Config struct { Debugging Debugging `json:"debugging"` // Internal-only fields - Ingress IngressConfig `json:"-"` + Ingress IngressConfig `json:"-"` + userSettings *Config `json:"-"` // the values read from the config file } +// NewMicroshiftConfig creates a new Config struct populated with the +// default values and with any computed values updated based on those +// defaults. func NewMicroshiftConfig() *Config { c := &Config{} - err := c.fillDefaults() - if err != nil { + if err := c.fillDefaults(); err != nil { + klog.Fatalf("Failed to initialize config: %v", err) + } + if err := c.updateComputedValues(); err != nil { klog.Fatalf("Failed to initialize config: %v", err) } return c } +// fillDefaults forcibly sets the configuration to the default +// values. We do not use a static struct for the defaults because some +// of them are computed from the environment. If any error occurs +// probing the environment, the values in the Config instance are not +// changed. func (c *Config) fillDefaults() error { - if c.Debugging.LogLevel == "" { - c.Debugging.LogLevel = "Normal" - } - if len(c.ApiServer.SubjectAltNames) == 0 { - subjectAltNames, err := getAllHostnames() - if err != nil { - return fmt.Errorf("failed to get all hostnames: %v", err) - } - c.ApiServer.SubjectAltNames = subjectAltNames + // Look up any values that may generate an error + subjectAltNames, err := getAllHostnames() + if err != nil { + return fmt.Errorf("failed to get all hostnames: %v", err) } - if c.ApiServer.URL == "" { - c.ApiServer.URL = "https://localhost:6443" + hostname, err := os.Hostname() + if err != nil { + return fmt.Errorf("Failed to get hostname %v", err) + } + nodeIP, err := util.GetHostIP() + if err != nil { + return fmt.Errorf("failed to get host IP: %v", err) } - if c.Node.HostnameOverride == "" { - nodeName, err := os.Hostname() - if err != nil { - return fmt.Errorf("Failed to get hostname %v", err) - } - c.Node.HostnameOverride = strings.ToLower(nodeName) + c.Debugging = Debugging{ + LogLevel: "Normal", } - if c.Node.NodeIP == "" { - nodeIP, err := util.GetHostIP() - if err != nil { - return fmt.Errorf("failed to get host IP: %v", err) - } - c.Node.NodeIP = nodeIP + c.ApiServer = ApiServer{ + SubjectAltNames: subjectAltNames, + URL: "https://localhost:6443", } - - if c.DNS.BaseDomain == "" { - c.DNS.BaseDomain = "example.com" + c.Node = Node{ + HostnameOverride: hostname, + NodeIP: nodeIP, } - - if len(c.Network.ClusterNetwork) == 0 { - c.Network.ClusterNetwork = []ClusterNetworkEntry{ + c.DNS = DNS{ + BaseDomain: "example.com", + } + c.Network = Network{ + ClusterNetwork: []ClusterNetworkEntry{ { CIDR: "10.42.0.0/16", }, - } - } - if len(c.Network.ServiceNetwork) == 0 { - c.Network.ServiceNetwork = []string{ + }, + ServiceNetwork: []string{ "10.43.0.0/16", + }, + ServiceNodePortRange: "30000-32767", + DNS: "10.43.0.10", + } + c.Etcd = EtcdConfig{ + MemoryLimitMB: 0, + DoStartupDefrag: true, // FIXME: flip this to SkipStartupDefrag + QuotaBackendBytes: 8 * 1024 * 1024 * 1024, + MinDefragBytes: 100 * 1024 * 1024, + MaxFragmentedPercentage: 45, + DefragCheckFreq: 5 * time.Minute, + } + + return nil +} + +// incorporateUserSettings merges any values read from the +// configuration file provided by the user with the existing settings +// (usually the defaults). +func (c *Config) incorporateUserSettings(u *Config) { + c.userSettings = u + + if u.DNS.BaseDomain != "" { + c.DNS.BaseDomain = u.DNS.BaseDomain + } + + if len(u.Network.ClusterNetwork) != 0 { + c.Network.ClusterNetwork = u.Network.ClusterNetwork + } + if len(u.Network.ServiceNetwork) != 0 { + c.Network.ServiceNetwork = u.Network.ServiceNetwork + // The default for the API server address is computed from the + // service network. If the user provides a network without + // also overriding the computed address, we need to clear the + // address here so it is recomputed later. If they provide + // both the network and the address, the address will be + // copied into place below with the other API server settings. + if u.ApiServer.AdvertiseAddress == "" { + c.ApiServer.AdvertiseAddress = "" } } - if c.Network.ServiceNodePortRange == "" { - c.Network.ServiceNodePortRange = "30000-32767" + if u.Network.ServiceNodePortRange != "" { + c.Network.ServiceNodePortRange = u.Network.ServiceNodePortRange + } + if u.Network.DNS != "" { + c.Network.DNS = u.Network.DNS + } + + if u.Etcd.MemoryLimitMB != 0 { + c.Etcd.MemoryLimitMB = u.Etcd.MemoryLimitMB + } + + if u.Node.HostnameOverride != "" { + c.Node.HostnameOverride = u.Node.HostnameOverride + } + if u.Node.NodeIP != "" { + c.Node.NodeIP = u.Node.NodeIP + } + + if len(u.ApiServer.SubjectAltNames) != 0 { + c.ApiServer.SubjectAltNames = u.ApiServer.SubjectAltNames + } + if u.ApiServer.AdvertiseAddress != "" { + c.ApiServer.AdvertiseAddress = u.ApiServer.AdvertiseAddress } - if c.Network.DNS == "" { - c.Network.DNS = "10.43.0.10" + if u.ApiServer.URL != "" { + c.ApiServer.URL = u.ApiServer.URL } - return c.updateComputedValues() + if u.Debugging.LogLevel != "" { + c.Debugging.LogLevel = u.Debugging.LogLevel + } } +// updateComputedValues examins the existing settings and converts any +// inputs to more easily consumable units or fills in any defaults +// computed based on the values of other settings. func (c *Config) updateComputedValues() error { + clusterDNS, err := c.computeClusterDNS() if err != nil { return err } c.Network.DNS = clusterDNS - // If KAS advertise address is not configured then compute it from the service - // CIDR automatically. + // If KAS advertise address configured, we do not want to apply + // the IP to the internal interface. + if c.userSettings != nil && len(c.userSettings.ApiServer.AdvertiseAddress) != 0 { + c.ApiServer.SkipInterface = true + } + + // If we have no advertise address, pick one. if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) @@ -124,9 +200,6 @@ func (c *Config) updateComputedValues() error { // First and last are the same because of the /32 netmask. firstValidIP, _ := cidr.AddressRange(nextSubnet) c.ApiServer.AdvertiseAddress = firstValidIP.String() - c.ApiServer.SkipInterface = false - } else { - c.ApiServer.SkipInterface = true } return nil diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1e8d8696d9..c447e805f4 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1,8 +1,10 @@ package config import ( + "fmt" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -24,13 +26,36 @@ func setupSuiteDataDir(t *testing.T) func() { } } -func TestParse(t *testing.T) { - mkConfig := func() *Config { +// TestGetActiveConfigFromYAML verifies that reading the config file +// correctly overrides the defaults and updates the computed values in +// the Config struct. +func TestGetActiveConfigFromYAML(t *testing.T) { + mkDefaultConfig := func() *Config { c := NewMicroshiftConfig() - c.ApiServer.SkipInterface = true return c } + dedent := func(input string) string { + lines := strings.Split(input, "\n") + detectIndentFrom := lines[0] + if detectIndentFrom == "" { + detectIndentFrom = lines[1] + } + dedentedLine := strings.TrimLeft(detectIndentFrom, " \t") + prefixLen := len(detectIndentFrom) - len(dedentedLine) + if prefixLen == 0 { + return input + } + var b strings.Builder + for _, line := range lines { + if len(line) >= prefixLen { + line = line[prefixLen:] + } + fmt.Fprintf(&b, "%s\n", line) + } + return b.String() + } + var ttests = []struct { name string config string @@ -40,32 +65,32 @@ func TestParse(t *testing.T) { { name: "empty", config: "", - expected: mkConfig(), + expected: mkDefaultConfig(), }, { name: "dns", - config: ` -dns: - baseDomain: test-example.com -`, + config: dedent(` + dns: + baseDomain: test-example.com + `), expected: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.DNS.BaseDomain = "test-example.com" return c }(), }, { name: "network", - config: ` -network: - clusterNetwork: - - cidr: "10.20.30.40/16" - serviceNetwork: - - "40.30.20.10/16" - serviceNodePortRange: "1024-32767" -`, + config: dedent(` + network: + clusterNetwork: + - cidr: "10.20.30.40/16" + serviceNetwork: + - "40.30.20.10/16" + serviceNodePortRange: "1024-32767" + `), expected: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.Network.ClusterNetwork = []ClusterNetworkEntry{ { CIDR: "10.20.30.40/16", @@ -73,62 +98,76 @@ network: } c.Network.ServiceNetwork = []string{"40.30.20.10/16"} c.Network.ServiceNodePortRange = "1024-32767" - c.ApiServer.AdvertiseAddress = "40.31.0.0" // computed default - c.updateComputedValues() // recomputes DNS field + c.ApiServer.AdvertiseAddress = "" // force value to be recomputed + c.updateComputedValues() // recomputes DNS field return c }(), }, { name: "node", - config: ` -node: - hostnameOverride: "node1" - nodeIP: "1.2.3.4" -`, + config: dedent(` + node: + hostnameOverride: "node1" + nodeIP: "1.2.3.4" + `), expected: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.Node.HostnameOverride = "node1" c.Node.NodeIP = "1.2.3.4" return c }(), }, { - name: "api-server", - config: ` -apiServer: - subjectAltNames: - - node1 - - node2 -`, + name: "api-server-subject-alt-names", + config: dedent(` + apiServer: + subjectAltNames: + - node1 + - node2 + `), expected: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.ApiServer.SubjectAltNames = []string{ "node1", "node2", } return c }(), }, + { + name: "api-server-advertise-address", + config: dedent(` + apiServer: + advertiseAddress: 4.3.2.1 + `), + expected: func() *Config { + c := mkDefaultConfig() + c.ApiServer.AdvertiseAddress = "4.3.2.1" + c.ApiServer.SkipInterface = true + return c + }(), + }, { name: "debugging", - config: ` -debugging: - logLevel: Info -`, + config: dedent(` + debugging: + logLevel: Info + `), expected: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.Debugging.LogLevel = "Info" return c }(), }, { name: "etcd", - config: ` -etcd: - memoryLimitMB: 100 -`, + config: dedent(` + etcd: + memoryLimitMB: 100 + `), expected: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.Etcd.MemoryLimitMB = 100 + c.updateComputedValues() return c }(), }, @@ -136,7 +175,9 @@ etcd: for _, tt := range ttests { t.Run(tt.name, func(t *testing.T) { - config, err := parse([]byte(tt.config)) + + config, err := getActiveConfigFromYAML([]byte(tt.config)) + if tt.expectErr && err == nil { t.Fatal("Expecting error and received nothing") } @@ -144,7 +185,13 @@ etcd: t.Fatalf("Not expecting error and received: %v", err) } if !tt.expectErr { - assert.Equal(t, tt.expected, config) + + // blank out the user settings because the expected value + // never has them and any computed value should be set so + // it should be safe to ignore them + config.userSettings = nil + + assert.Equal(t, tt.expected, config, "config input:\n---%s\n---", tt.config) } }) } @@ -155,7 +202,7 @@ func TestValidate(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() - mkConfig := func() *Config { + mkDefaultConfig := func() *Config { c := NewMicroshiftConfig() c.ApiServer.SkipInterface = true return c @@ -174,7 +221,7 @@ func TestValidate(t *testing.T) { { name: "subject-alt-names-with-localhost", config: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.ApiServer.SubjectAltNames = []string{"localhost"} return c }(), @@ -183,7 +230,7 @@ func TestValidate(t *testing.T) { { name: "subject-alt-names-with-loopback-ipv4", config: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.ApiServer.SubjectAltNames = []string{"127.0.0.1"} return c }(), @@ -192,7 +239,7 @@ func TestValidate(t *testing.T) { { name: "subject-alt-names-with-kubernetes", config: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.ApiServer.SubjectAltNames = []string{"kubernetes"} return c }(), @@ -201,7 +248,7 @@ func TestValidate(t *testing.T) { { name: "etcd-memory-limit-low", config: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.Etcd.MemoryLimitMB = 1 return c }(), @@ -210,7 +257,7 @@ func TestValidate(t *testing.T) { { name: "etcd-memory-zero", config: func() *Config { - c := mkConfig() + c := mkDefaultConfig() c.Etcd.MemoryLimitMB = 0 return c }(), diff --git a/pkg/config/files.go b/pkg/config/files.go index afeca6fa12..3b8abfd44b 100644 --- a/pkg/config/files.go +++ b/pkg/config/files.go @@ -76,35 +76,38 @@ func findManifestsDir() []string { func parse(contents []byte) (*Config, error) { c := &Config{} - fmt.Printf("parsing %s\n", string(contents)) if err := yaml.Unmarshal(contents, c); err != nil { return nil, fmt.Errorf("Unable to decode configuration: %v", err) } - if err := c.fillDefaults(); err != nil { - return nil, fmt.Errorf("Invalid configuration: %v", err) + return c, nil +} + +func getActiveConfigFromYAML(contents []byte) (*Config, error) { + userSettings, err := parse(contents) + if err != nil { + return nil, fmt.Errorf("Error parsing config file %q: %v", configFile, err) } - if err := c.updateComputedValues(); err != nil { + + // Start with the defaults, then apply the user settings and + // recompute dynamic values. + results := &Config{} + if err := results.fillDefaults(); err != nil { return nil, fmt.Errorf("Invalid configuration: %v", err) } - if err := c.validate(); err != nil { + results.incorporateUserSettings(userSettings) + if err := results.updateComputedValues(); err != nil { return nil, fmt.Errorf("Invalid configuration: %v", err) } - return c, nil -} - -func Read(configFile string) (*Config, error) { - contents, err := os.ReadFile(configFile) - if err != nil { - return nil, fmt.Errorf("reading config file %q: %v", configFile, err) + if err := results.validate(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - return parse(contents) + return results, nil } // Get the active configuration. If the configuration file exists, // read it and require it to be valid. Otherwise return the default // settings. func GetActiveConfig() (*Config, error) { - var cfg *Config filename := GetConfigFile() _, err := os.Stat(filename) if os.IsNotExist(err) { @@ -113,9 +116,11 @@ func GetActiveConfig() (*Config, error) { } else if err != nil { return nil, err } - cfg, err = Read(filename) + + // Read the file and merge user-provided settings with the defaults + contents, err := os.ReadFile(configFile) if err != nil { - return nil, err + return nil, fmt.Errorf("Error reading config file %q: %v", configFile, err) } - return cfg, nil + return getActiveConfigFromYAML(contents) } From b022ede455391860f84f8e25129b42a24b505e61 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 10 Mar 2023 12:41:10 -0500 Subject: [PATCH 12/17] USHIFT-936: rename config entry point functions Use clearer names based on what they return. --- etcd/cmd/microshift-etcd/run.go | 2 +- .../openshift/microshift/pkg/config/config.go | 4 ++-- .../openshift/microshift/pkg/config/files.go | 10 +++++----- pkg/cmd/run.go | 2 +- pkg/cmd/showConfig.go | 4 ++-- pkg/components/render_test.go | 2 +- pkg/config/config.go | 4 ++-- pkg/config/config_test.go | 14 +++++++------- pkg/config/debugging_test.go | 2 +- pkg/config/files.go | 10 +++++----- pkg/controllers/kube-controller-manager_test.go | 2 +- 11 files changed, 28 insertions(+), 28 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index 34f7b1093a..8aa6fe3e17 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -25,7 +25,7 @@ func NewRunEtcdCommand() *cobra.Command { cmd := &cobra.Command{ Use: "run", RunE: func(cmd *cobra.Command, args []string) (err error) { - cfg, err := config.GetActiveConfig() + cfg, err := config.ActiveConfig() if err != nil { klog.Fatalf("Error in reading and validating MicroShift config: %v", err) } diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 7939ee5858..3a9739a79a 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -35,10 +35,10 @@ type Config struct { userSettings *Config `json:"-"` // the values read from the config file } -// NewMicroshiftConfig creates a new Config struct populated with the +// NewDefault creates a new Config struct populated with the // default values and with any computed values updated based on those // defaults. -func NewMicroshiftConfig() *Config { +func NewDefault() *Config { c := &Config{} if err := c.fillDefaults(); err != nil { klog.Fatalf("Failed to initialize config: %v", err) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go index 3b8abfd44b..6c78397125 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go @@ -104,15 +104,15 @@ func getActiveConfigFromYAML(contents []byte) (*Config, error) { return results, nil } -// Get the active configuration. If the configuration file exists, -// read it and require it to be valid. Otherwise return the default -// settings. -func GetActiveConfig() (*Config, error) { +// ActiveConfig returns the active configuration. If the configuration +// file exists, read it and require it to be valid. Otherwise return +// the default settings. +func ActiveConfig() (*Config, error) { filename := GetConfigFile() _, err := os.Stat(filename) if os.IsNotExist(err) { // No configuration file, use the default settings - return NewMicroshiftConfig(), nil + return NewDefault(), nil } else if err != nil { return nil, err } diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 1cfc08eaed..b5fe89b9fa 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -34,7 +34,7 @@ func NewRunMicroshiftCommand() *cobra.Command { Use: "run", Short: "Run MicroShift", RunE: func(cmd *cobra.Command, args []string) error { - cfg, err := config.GetActiveConfig() + cfg, err := config.ActiveConfig() if err != nil { return err } diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 2a90932b19..1f92352fd5 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -28,12 +28,12 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command var err error if opts.Mode == "effective" { - cfg, err = config.GetActiveConfig() + cfg, err = config.ActiveConfig() if err != nil { cmdutil.CheckErr(err) } } else { - cfg = config.NewMicroshiftConfig() + cfg = config.NewDefault() } marshalled, err := yaml.Marshal(cfg) diff --git a/pkg/components/render_test.go b/pkg/components/render_test.go index 5498cfc65c..33b1e1fb88 100644 --- a/pkg/components/render_test.go +++ b/pkg/components/render_test.go @@ -101,7 +101,7 @@ func Test_renderTopolvmDaemonsetTemplate(t *testing.T) { name: "renders lvmd-socket-name path", args: args{ tb: tb, - data: renderParamsFromConfig(config.NewMicroshiftConfig(), assets.RenderParams{"SocketName": "/run/lvmd/lvmd.socket", "lvmd": "foobar"}), + data: renderParamsFromConfig(config.NewDefault(), assets.RenderParams{"SocketName": "/run/lvmd/lvmd.socket", "lvmd": "foobar"}), }, want: wantBytes(tpl, map[string]interface{}{ "ReleaseImage": release.Image, diff --git a/pkg/config/config.go b/pkg/config/config.go index 7939ee5858..3a9739a79a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -35,10 +35,10 @@ type Config struct { userSettings *Config `json:"-"` // the values read from the config file } -// NewMicroshiftConfig creates a new Config struct populated with the +// NewDefault creates a new Config struct populated with the // default values and with any computed values updated based on those // defaults. -func NewMicroshiftConfig() *Config { +func NewDefault() *Config { c := &Config{} if err := c.fillDefaults(); err != nil { klog.Fatalf("Failed to initialize config: %v", err) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index c447e805f4..b4f486332e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -31,7 +31,7 @@ func setupSuiteDataDir(t *testing.T) func() { // the Config struct. func TestGetActiveConfigFromYAML(t *testing.T) { mkDefaultConfig := func() *Config { - c := NewMicroshiftConfig() + c := NewDefault() return c } @@ -203,7 +203,7 @@ func TestValidate(t *testing.T) { defer cleanup() mkDefaultConfig := func() *Config { - c := NewMicroshiftConfig() + c := NewDefault() c.ApiServer.SkipInterface = true return c } @@ -215,7 +215,7 @@ func TestValidate(t *testing.T) { }{ { name: "defaults-ok", - config: NewMicroshiftConfig(), + config: NewDefault(), expectErr: false, }, { @@ -278,7 +278,7 @@ func TestValidate(t *testing.T) { } func TestMicroshiftConfigIsDefaultNodeName(t *testing.T) { - c := NewMicroshiftConfig() + c := NewDefault() if !c.isDefaultNodeName() { t.Errorf("expected default IsDefaultNodeName to be true") } @@ -293,7 +293,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() - c := NewMicroshiftConfig() + c := NewDefault() c.Node.HostnameOverride = "node1" if err := c.validateNodeName(IS_NOT_DEFAULT_NODENAME); err != nil { @@ -321,7 +321,7 @@ func TestMicroshiftConfigNodeNameValidationFromDefault(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() - c := NewMicroshiftConfig() + c := NewDefault() if err := c.validateNodeName(IS_DEFAULT_NODENAME); err != nil { t.Errorf("failed to validate node name on first call: %v", err) @@ -349,7 +349,7 @@ func TestMicroshiftConfigNodeNameValidationBadName(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() - c := NewMicroshiftConfig() + c := NewDefault() c.Node.HostnameOverride = "1.2.3.4" if err := c.validateNodeName(IS_DEFAULT_NODENAME); err == nil { diff --git a/pkg/config/debugging_test.go b/pkg/config/debugging_test.go index 2a934370e4..4ac79df68b 100644 --- a/pkg/config/debugging_test.go +++ b/pkg/config/debugging_test.go @@ -39,7 +39,7 @@ func TestGetVerbosity(t *testing.T) { for _, tt := range ttests { t.Run(tt.setting, func(t *testing.T) { - config := NewMicroshiftConfig() + config := NewDefault() config.Debugging.LogLevel = tt.setting verbosity := config.GetVerbosity() assert.Equal(t, tt.level, verbosity) diff --git a/pkg/config/files.go b/pkg/config/files.go index 3b8abfd44b..6c78397125 100644 --- a/pkg/config/files.go +++ b/pkg/config/files.go @@ -104,15 +104,15 @@ func getActiveConfigFromYAML(contents []byte) (*Config, error) { return results, nil } -// Get the active configuration. If the configuration file exists, -// read it and require it to be valid. Otherwise return the default -// settings. -func GetActiveConfig() (*Config, error) { +// ActiveConfig returns the active configuration. If the configuration +// file exists, read it and require it to be valid. Otherwise return +// the default settings. +func ActiveConfig() (*Config, error) { filename := GetConfigFile() _, err := os.Stat(filename) if os.IsNotExist(err) { // No configuration file, use the default settings - return NewMicroshiftConfig(), nil + return NewDefault(), nil } else if err != nil { return nil, err } diff --git a/pkg/controllers/kube-controller-manager_test.go b/pkg/controllers/kube-controller-manager_test.go index da2c9aae2f..110e31e486 100644 --- a/pkg/controllers/kube-controller-manager_test.go +++ b/pkg/controllers/kube-controller-manager_test.go @@ -37,7 +37,7 @@ func TestKCMDefaultConfigAsset(t *testing.T) { } func TestConfigure(t *testing.T) { - cfg := config.NewMicroshiftConfig() + cfg := config.NewDefault() kcm := NewKubeControllerManager(cfg) clusterSigningKey, clusterSigningCert := kcmClusterSigningCertKeyAndFile() From cdc051d9adbcb7e5131c18a292f554e86ab42700 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 8 Mar 2023 15:45:01 -0500 Subject: [PATCH 13/17] USHIFT-936: introduce CanonicalNodeName We need to convert the hostname to a value suitable for use as a node name. Instead of doing that in many places, provide a method to return the modified value. --- .../openshift/microshift/pkg/config/node.go | 25 ++++++++++----- pkg/cmd/init.go | 2 +- pkg/components/render.go | 2 +- pkg/config/config_test.go | 31 +++++++++++++++++++ pkg/config/node.go | 25 ++++++++++----- 5 files changed, 67 insertions(+), 18 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go index cdd6a745dc..13eb6e6a90 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go @@ -25,20 +25,28 @@ func (c *Config) isDefaultNodeName() bool { if err != nil { klog.Fatalf("Failed to get hostname %v", err) } - return c.Node.HostnameOverride == strings.ToLower(hostname) + return c.CanonicalNodeName() == strings.ToLower(hostname) +} + +// CanonicalNodeName returns the name to use for the node. The value +// is taken from either the HostnameOverride provided by the user in +// the config file, or the host name. +func (c *Config) CanonicalNodeName() string { + return strings.ToLower(c.Node.HostnameOverride) } // Read or set the NodeName that will be used for this MicroShift instance func (c *Config) establishNodeName() (string, error) { + name := c.CanonicalNodeName() filePath := filepath.Join(GetDataDir(), ".nodename") contents, err := os.ReadFile(filePath) if os.IsNotExist(err) { // ensure that dataDir exists os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { + if err := os.WriteFile(filePath, []byte(name), 0444); err != nil { return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) } - return c.Node.HostnameOverride, nil + return name, nil } else if err != nil { return "", err } @@ -47,8 +55,9 @@ func (c *Config) establishNodeName() (string, error) { // Validate the NodeName to be used for this MicroShift instances func (c *Config) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) + currentNodeName := c.CanonicalNodeName() + if addr := net.ParseIP(currentNodeName); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", currentNodeName) } establishedNodeName, err := c.establishNodeName() @@ -56,14 +65,14 @@ func (c *Config) validateNodeName(isDefaultNodeName bool) error { return fmt.Errorf("failed to establish NodeName: %v", err) } - if establishedNodeName != c.Node.HostnameOverride { + if establishedNodeName != currentNodeName { if !isDefaultNodeName { return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.Node.HostnameOverride, establishedNodeName) + currentNodeName, establishedNodeName) } else { c.Node.HostnameOverride = establishedNodeName klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) + "Please consider using a static NodeName in configuration", establishedNodeName) } } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index b7a1df47b3..b1e28231dc 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -172,7 +172,7 @@ func certSetup(cfg *config.Config) (*certchains.CertificateChains, error) { ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, // userinfo per https://kubernetes.io/docs/reference/access-authn-authz/node/#overview - UserInfo: &user.DefaultInfo{Name: "system:node:" + cfg.Node.HostnameOverride, Groups: []string{"system:nodes"}}, + UserInfo: &user.DefaultInfo{Name: "system:node:" + cfg.CanonicalNodeName(), Groups: []string{"system:nodes"}}, }, ).WithServingCertificates( &certchains.ServingCertificateSigningRequestInfo{ diff --git a/pkg/components/render.go b/pkg/components/render.go index 5511218cf8..fd616d2fde 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -23,7 +23,7 @@ var templateFuncs = map[string]interface{}{ func renderParamsFromConfig(cfg *config.Config, extra assets.RenderParams) assets.RenderParams { params := map[string]interface{}{ "ReleaseImage": release.Image, - "NodeName": cfg.Node.HostnameOverride, + "NodeName": cfg.CanonicalNodeName(), "NodeIP": cfg.Node.NodeIP, "ClusterCIDR": cfg.Network.ClusterNetwork[0].CIDR, "ServiceCIDR": cfg.Network.ServiceNetwork[0], diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index b4f486332e..fc57668286 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -289,6 +289,37 @@ func TestMicroshiftConfigIsDefaultNodeName(t *testing.T) { } } +func TestCanonicalNodeName(t *testing.T) { + hostname, _ := os.Hostname() + + var ttests = []struct { + name string + value string + expected string + }{ + { + name: "default", + value: "", + expected: strings.ToLower(hostname), + }, + { + name: "upper-case", + value: "Hostname", + expected: "hostname", + }, + } + + for _, tt := range ttests { + t.Run(tt.name, func(t *testing.T) { + c := NewDefault() + if tt.value != "" { // account for default + c.Node.HostnameOverride = tt.value + } + assert.Equal(t, tt.expected, c.CanonicalNodeName()) + }) + } +} + func TestMicroshiftConfigNodeNameValidation(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() diff --git a/pkg/config/node.go b/pkg/config/node.go index cdd6a745dc..13eb6e6a90 100644 --- a/pkg/config/node.go +++ b/pkg/config/node.go @@ -25,20 +25,28 @@ func (c *Config) isDefaultNodeName() bool { if err != nil { klog.Fatalf("Failed to get hostname %v", err) } - return c.Node.HostnameOverride == strings.ToLower(hostname) + return c.CanonicalNodeName() == strings.ToLower(hostname) +} + +// CanonicalNodeName returns the name to use for the node. The value +// is taken from either the HostnameOverride provided by the user in +// the config file, or the host name. +func (c *Config) CanonicalNodeName() string { + return strings.ToLower(c.Node.HostnameOverride) } // Read or set the NodeName that will be used for this MicroShift instance func (c *Config) establishNodeName() (string, error) { + name := c.CanonicalNodeName() filePath := filepath.Join(GetDataDir(), ".nodename") contents, err := os.ReadFile(filePath) if os.IsNotExist(err) { // ensure that dataDir exists os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { + if err := os.WriteFile(filePath, []byte(name), 0444); err != nil { return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) } - return c.Node.HostnameOverride, nil + return name, nil } else if err != nil { return "", err } @@ -47,8 +55,9 @@ func (c *Config) establishNodeName() (string, error) { // Validate the NodeName to be used for this MicroShift instances func (c *Config) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) + currentNodeName := c.CanonicalNodeName() + if addr := net.ParseIP(currentNodeName); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", currentNodeName) } establishedNodeName, err := c.establishNodeName() @@ -56,14 +65,14 @@ func (c *Config) validateNodeName(isDefaultNodeName bool) error { return fmt.Errorf("failed to establish NodeName: %v", err) } - if establishedNodeName != c.Node.HostnameOverride { + if establishedNodeName != currentNodeName { if !isDefaultNodeName { return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.Node.HostnameOverride, establishedNodeName) + currentNodeName, establishedNodeName) } else { c.Node.HostnameOverride = establishedNodeName klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) + "Please consider using a static NodeName in configuration", establishedNodeName) } } From 23272d233c7bad6716d90541f490981bffb628a8 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 10 Mar 2023 16:49:57 -0500 Subject: [PATCH 14/17] USHIFT-936: remove Etcd.DoStartupDefrag boolean We always want to defragment on startup, so we do not need a flag. --- etcd/cmd/microshift-etcd/run.go | 14 +++++--------- .../openshift/microshift/pkg/config/config.go | 1 - .../openshift/microshift/pkg/config/etcd.go | 6 +----- pkg/config/config.go | 1 - pkg/config/etcd.go | 6 +----- 5 files changed, 7 insertions(+), 21 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index 8aa6fe3e17..a9a56de9c9 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -52,7 +52,6 @@ type EtcdService struct { minDefragBytes int64 maxFragmentedPercentage float64 defragCheckFreq time.Duration - doStartupDefrag bool } func NewEtcd(cfg *config.Config) *EtcdService { @@ -67,7 +66,6 @@ func (s *EtcdService) configure(cfg *config.Config) { s.minDefragBytes = cfg.Etcd.MinDefragBytes s.maxFragmentedPercentage = cfg.Etcd.MaxFragmentedPercentage s.defragCheckFreq = cfg.Etcd.DefragCheckFreq - s.doStartupDefrag = cfg.Etcd.DoStartupDefrag microshiftDataDir := config.GetDataDir() certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) @@ -120,13 +118,11 @@ func (s *EtcdService) Run() error { <-e.Server.StopNotify() }() - // If we were told to, go ahead and do a defragment now. - if s.doStartupDefrag { - if err := e.Server.Backend().Defrag(); err != nil { - err = fmt.Errorf("initial defragmentation failed: %v", err) - klog.Error(err) - return err - } + // Go ahead and do a defragment now. + if err := e.Server.Backend().Defrag(); err != nil { + err = fmt.Errorf("initial defragmentation failed: %v", err) + klog.Error(err) + return err } // Start up the defrag controller. diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 3a9739a79a..642c09efa5 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -98,7 +98,6 @@ func (c *Config) fillDefaults() error { } c.Etcd = EtcdConfig{ MemoryLimitMB: 0, - DoStartupDefrag: true, // FIXME: flip this to SkipStartupDefrag QuotaBackendBytes: 8 * 1024 * 1024 * 1024, MinDefragBytes: 100 * 1024 * 1024, MaxFragmentedPercentage: 45, diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go index dc26b92ec2..cb571eb963 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go @@ -24,10 +24,6 @@ type EtcdConfig struct { MaxFragmentedPercentage float64 `json:"-"` // How often to check the conditions for defragging (0 means no - // defrags, except for a single on startup if `doStartupDefrag` is - // set). + // defrags, except for a single on startup). DefragCheckFreq time.Duration `json:"-"` - - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool `json:"-"` } diff --git a/pkg/config/config.go b/pkg/config/config.go index 3a9739a79a..642c09efa5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -98,7 +98,6 @@ func (c *Config) fillDefaults() error { } c.Etcd = EtcdConfig{ MemoryLimitMB: 0, - DoStartupDefrag: true, // FIXME: flip this to SkipStartupDefrag QuotaBackendBytes: 8 * 1024 * 1024 * 1024, MinDefragBytes: 100 * 1024 * 1024, MaxFragmentedPercentage: 45, diff --git a/pkg/config/etcd.go b/pkg/config/etcd.go index dc26b92ec2..cb571eb963 100644 --- a/pkg/config/etcd.go +++ b/pkg/config/etcd.go @@ -24,10 +24,6 @@ type EtcdConfig struct { MaxFragmentedPercentage float64 `json:"-"` // How often to check the conditions for defragging (0 means no - // defrags, except for a single on startup if `doStartupDefrag` is - // set). + // defrags, except for a single on startup). DefragCheckFreq time.Duration `json:"-"` - - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool `json:"-"` } From af48087ececd2162092e9bb6b48ee0351cdc9730 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 13 Mar 2023 10:04:32 -0400 Subject: [PATCH 15/17] fix formatting of comments --- pkg/controllers/etcd.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/controllers/etcd.go b/pkg/controllers/etcd.go index dee64858dd..3afadf6b4a 100644 --- a/pkg/controllers/etcd.go +++ b/pkg/controllers/etcd.go @@ -61,13 +61,13 @@ func (s *EtcdService) Run(ctx context.Context, ready chan<- struct{}, stopped ch return fmt.Errorf("%v failed to get exec path: %v", s.Name(), err) } etcdPath := filepath.Join(filepath.Dir(microshiftExecPath), "microshift-etcd") - // Not running the etcd binary directly, the proper etcd arguments are handled - // in etcd/cmd/microshift-etcd/run.go. + // Not running the etcd binary directly, the proper etcd arguments + // are handled in etcd/cmd/microshift-etcd/run.go. args := []string{} - // If we're launching MicroShift as a service, we need to do the same - // with etcd, so wrap it in a transient systemd-unit that's tied - // to the MicroShift service lifetime. + // If we're launching MicroShift as a service, we need to do the + // same with etcd, so wrap it in a transient systemd-unit that's + // tied to the MicroShift service lifetime. var exe string if runningAsSvc { args = append(args, From 29dc7987f14bffbea95d46cbf70e546d4318bcf7 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 14 Mar 2023 11:12:12 -0400 Subject: [PATCH 16/17] USHIFT-936: simplify API server port handling The URL field is not user-configurable, so we don't need a function to parse it. --- .../microshift/pkg/config/apiserver.go | 31 ++----------------- .../openshift/microshift/pkg/config/config.go | 1 + pkg/cmd/init.go | 7 ++--- pkg/config/apiserver.go | 31 ++----------------- pkg/config/config.go | 1 + pkg/controllers/kube-apiserver.go | 8 +---- 6 files changed, 11 insertions(+), 68 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go index dcb398c7a0..88623d03e9 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go @@ -1,10 +1,5 @@ package config -import ( - "net/url" - "strconv" -) - type ApiServer struct { // SubjectAltNames added to API server certs SubjectAltNames []string `json:"subjectAltNames"` @@ -18,27 +13,7 @@ type ApiServer struct { // AdvertiseAddress in the loopback interface. Automatically computed. SkipInterface bool `json:"-"` - // The URL of the API server - URL string `json:"-"` -} - -// extract the api server port from the cluster URL -func (c *Config) ApiServerPort() (int, error) { - var port string - - parsed, err := url.Parse(c.ApiServer.URL) - if err != nil { - return 0, err - } - - // default empty URL to port 6443 - port = parsed.Port() - if port == "" { - port = "6443" - } - portNum, err := strconv.Atoi(port) - if err != nil { - return 0, err - } - return portNum, nil + // The URL and Port of the API server cannot be changed by the user. + URL string `json:"-"` + Port int `json:"-"` } diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 642c09efa5..26e0c71b40 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -76,6 +76,7 @@ func (c *Config) fillDefaults() error { c.ApiServer = ApiServer{ SubjectAltNames: subjectAltNames, URL: "https://localhost:6443", + Port: 6443, } c.Node = Node{ HostnameOverride: hostname, diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index b1e28231dc..864e785b0f 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -22,6 +22,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "time" "k8s.io/apiserver/pkg/authentication/serviceaccount" @@ -387,14 +388,10 @@ func initKubeconfigs( if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } - apiServerPort, err := cfg.ApiServerPort() - if err != nil { - return fmt.Errorf("failed to get apiserver port: %v", err) - } // Generate one kubeconfigs per name for _, name := range append(cfg.ApiServer.SubjectAltNames, cfg.Node.HostnameOverride) { - u.Host = fmt.Sprintf("%s:%d", name, apiServerPort) + u.Host = net.JoinHostPort(name, strconv.Itoa(cfg.ApiServer.Port)) if err := util.KubeConfigWithClientCerts( cfg.KubeConfigAdminPath(name), u.String(), diff --git a/pkg/config/apiserver.go b/pkg/config/apiserver.go index dcb398c7a0..88623d03e9 100644 --- a/pkg/config/apiserver.go +++ b/pkg/config/apiserver.go @@ -1,10 +1,5 @@ package config -import ( - "net/url" - "strconv" -) - type ApiServer struct { // SubjectAltNames added to API server certs SubjectAltNames []string `json:"subjectAltNames"` @@ -18,27 +13,7 @@ type ApiServer struct { // AdvertiseAddress in the loopback interface. Automatically computed. SkipInterface bool `json:"-"` - // The URL of the API server - URL string `json:"-"` -} - -// extract the api server port from the cluster URL -func (c *Config) ApiServerPort() (int, error) { - var port string - - parsed, err := url.Parse(c.ApiServer.URL) - if err != nil { - return 0, err - } - - // default empty URL to port 6443 - port = parsed.Port() - if port == "" { - port = "6443" - } - portNum, err := strconv.Atoi(port) - if err != nil { - return 0, err - } - return portNum, nil + // The URL and Port of the API server cannot be changed by the user. + URL string `json:"-"` + Port int `json:"-"` } diff --git a/pkg/config/config.go b/pkg/config/config.go index 642c09efa5..26e0c71b40 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -76,6 +76,7 @@ func (c *Config) fillDefaults() error { c.ApiServer = ApiServer{ SubjectAltNames: subjectAltNames, URL: "https://localhost:6443", + Port: 6443, } c.Node = Node{ HostnameOverride: hostname, diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index d9b38fbe3f..96fbcead51 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -109,12 +109,6 @@ func (s *KubeAPIServer) configure(cfg *config.Config) error { return fmt.Errorf("failed to configure kube-apiserver audit policy: %w", err) } - // Get the apiserver port so we can set it as an argument - apiServerPort, err := cfg.ApiServerPort() - if err != nil { - return err - } - s.masterURL = cfg.ApiServer.URL s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir) s.advertiseAddress = cfg.ApiServer.AdvertiseAddress @@ -185,7 +179,7 @@ func (s *KubeAPIServer) configure(cfg *config.Config) error { }, ServingInfo: configv1.HTTPServingInfo{ ServingInfo: configv1.ServingInfo{ - BindAddress: net.JoinHostPort("0.0.0.0", strconv.Itoa(apiServerPort)), + BindAddress: net.JoinHostPort("0.0.0.0", strconv.Itoa(cfg.ApiServer.Port)), MinTLSVersion: string(fixedTLSProfile.MinTLSVersion), CipherSuites: crypto.OpenSSLToIANACipherSuites(fixedTLSProfile.Ciphers), NamedCertificates: []configv1.NamedCertificate{ From b0d33719d7b1112681d32798d8b5fee5532ff147 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 25 Mar 2023 14:32:27 -0400 Subject: [PATCH 17/17] ETCD-403: log command line for starting etcd --- pkg/controllers/etcd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controllers/etcd.go b/pkg/controllers/etcd.go index 3afadf6b4a..d875652207 100644 --- a/pkg/controllers/etcd.go +++ b/pkg/controllers/etcd.go @@ -89,6 +89,7 @@ func (s *EtcdService) Run(ctx context.Context, ready chan<- struct{}, stopped ch } args = append(args, "run") // Not using context as canceling ctx sends SIGKILL to process + klog.Infof("starting etcd via %s with args %v", exe, args) cmd := exec.Command(exe, args...) wd, err := os.Getwd()